diff --git a/tierb_study_20260826_144644/STATUS.txt b/tierb_study_20260826_144644/STATUS.txt index 355d5862c11c8ec10b72a3cb053f637fccd11396..0368e82910f3a95db07df387e08f9e4ca36c11c8 100644 --- a/tierb_study_20260826_144644/STATUS.txt +++ b/tierb_study_20260826_144644/STATUS.txt @@ -1,12 +1,9 @@ study_root: /workspace/tierb_study_20260826_144644 gbs: 8 -updated: 2026-08-26T16:00:42+00:00 -current: echo__tierb__gbs8 +updated: 2026-08-26T16:16:43+00:00 +current: echo__tierbprobe__gbs8 completed steps: - echo__tierbprobe__gbs8 - moonep__tierbprobe__gbs8 - ultraep__tierbprobe__gbs8 failed steps: FAIL ultraep tierbprobe 8 2026-08-26T14:57:57+00:00 2026-08-26T15:05:21+00:00 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..9e14460c806f202c8585bbbbaa9badb60a1299cf --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 2 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 5833 +1, 5857 +2, 6015 +3, 6167 +4, 5869 +5, 6015 +6, 6015 +7, 5993 +0, 26681 +1, 26761 +2, 26635 +3, 26625 +4, 26611 +5, 26639 +6, 26777 +7, 26767 +0, 36379 +1, 36457 +2, 36323 +3, 36323 +4, 36301 +5, 36317 +6, 36455 +7, 36477 +0, 37785 +1, 37869 +2, 37719 +3, 37717 +4, 37717 +5, 37715 +6, 37851 +7, 37869 +0, 37873 +1, 37957 +2, 37807 +3, 37805 +4, 37805 +5, 37803 +6, 37939 +7, 37957 +0, 37937 +1, 38037 +2, 37871 +3, 37883 +4, 37877 +5, 37889 +6, 38017 +7, 38051 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38827 +1, 38911 +2, 38761 +3, 38761 +4, 38761 +5, 38757 +6, 38893 +7, 38911 +0, 38829 +1, 38915 +2, 38763 +3, 38763 +4, 38763 +5, 38761 +6, 38895 +7, 38915 +0, 121367 +1, 121723 +2, 121507 +3, 121477 +4, 121505 +5, 121611 +6, 121651 +7, 121683 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 122541 +1, 122837 +2, 122667 +3, 124717 +4, 122667 +5, 122747 +6, 122817 +7, 122837 +0, 123719 +1, 126063 +2, 125893 +3, 125893 +4, 125893 +5, 125973 +6, 126043 +7, 126063 +0, 124369 +1, 126725 +2, 126521 +3, 126487 +4, 126511 +5, 126599 +6, 126665 +7, 126673 +0, 97651 +1, 99299 +2, 99913 +3, 99737 +4, 104029 +5, 99213 +6, 99287 +7, 99309 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 2058 +1, 1034 +2, 169343 +3, 2314 +4, 169343 +5, 168633 +6, 168709 +7, 168721 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fa737515ef44118e6b0539752315584ad6ff5246 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8f98fedda7d5d376a6a5c05b184572c8167d6453 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/train.log @@ -0,0 +1,1572 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:09:06.057000 189383 torch/distributed/run.py:766] +W0826 16:09:06.057000 189383 torch/distributed/run.py:766] ***************************************** +W0826 16:09:06.057000 189383 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:09:06.057000 189383 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 2 is connected to 71 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 43201 is connected to is connected to is connected to [Gloo] Rank is connected to is connected to 777577 peer ranks. peer ranks. peer ranks. is connected to peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 peer ranks. 77 + + + +Expected number of connected peer ranks is : +7 +[Gloo] Rank 7 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 76 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to [Gloo] Rank 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 16:09:24.166692523 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:09:24.167401444 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:09:24.167419039 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:09:24.169040807 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:09:24.169040609 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:09:24.169073097 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:09:24.169073996 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:09:24.320341368 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.733 seconds +time to initialize megatron (seconds): 6.949 +[after megatron is initialized] datetime: 2026-08-26 16:09:29 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.input_layernorm.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:09:30 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:09:30 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (637.34, 647.52) + train/valid/test-data-iterators-setup ..........: (60.00, 67.56) +[before the start of training step] datetime: 2026-08-26 16:09:30 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:10:09] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38882.2 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 40525.23095703125 | max allocated: 115427.2763671875 | reserved: 117354.0 | max reserved: 117354.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:10:10] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 647.5 | throughput per GPU (TFLOP/s/GPU): 197.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:10] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 648.1 | throughput per GPU (TFLOP/s/GPU): 196.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:10:12] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1991.9 | throughput per GPU (TFLOP/s/GPU): 64.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:13] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 597.5 | throughput per GPU (TFLOP/s/GPU): 213.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:14] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 602.8 | throughput per GPU (TFLOP/s/GPU): 211.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:14] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 605.7 | throughput per GPU (TFLOP/s/GPU): 210.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.975649E+00 | loss scale: 1.0 | grad norm: 3.681 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:15] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 600.2 | throughput per GPU (TFLOP/s/GPU): 212.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785368E+00 | loss scale: 1.0 | grad norm: 5.697 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:15] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 598.5 | throughput per GPU (TFLOP/s/GPU): 213.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.613759E+00 | loss scale: 1.0 | grad norm: 3.002 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:16] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 598.1 | throughput per GPU (TFLOP/s/GPU): 213.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.899922E+00 | loss scale: 1.0 | grad norm: 1155.511 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:17] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 598.5 | throughput per GPU (TFLOP/s/GPU): 213.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.563369E+00 | loss scale: 1.0 | grad norm: 14.007 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:10:17] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 600.2 | throughput per GPU (TFLOP/s/GPU): 212.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.509861E+00 | loss scale: 1.0 | grad norm: 83.626 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:10:17 +[rank0]:[W826 16:10:19.055436364 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..b423b255d59346dc78782ec0a0f799331d87f848 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 823 +1, 829 +2, 827 +3, 829 +4, 833 +5, 829 +6, 833 +7, 837 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 26289 +1, 26377 +2, 26223 +3, 26223 +4, 26223 +5, 26223 +6, 26355 +7, 26377 +0, 36251 +1, 36335 +2, 36185 +3, 36183 +4, 36183 +5, 36181 +6, 36317 +7, 36335 +0, 36901 +1, 36985 +2, 36835 +3, 36833 +4, 36833 +5, 36831 +6, 36967 +7, 36985 +0, 37785 +1, 37869 +2, 37719 +3, 37717 +4, 37717 +5, 37715 +6, 37851 +7, 37869 +0, 37873 +1, 37957 +2, 37807 +3, 37805 +4, 37805 +5, 37803 +6, 37939 +7, 37957 +0, 38051 +1, 38133 +2, 37993 +3, 37997 +4, 37995 +5, 38007 +6, 38161 +7, 38143 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38821 +1, 38905 +2, 38755 +3, 38755 +4, 38755 +5, 38751 +6, 38887 +7, 38905 +0, 38829 +1, 38915 +2, 38763 +3, 38763 +4, 38763 +5, 38761 +6, 38895 +7, 38915 +0, 38829 +1, 38915 +2, 38763 +3, 38763 +4, 38763 +5, 38761 +6, 38895 +7, 38915 +0, 121325 +1, 121647 +2, 121475 +3, 121461 +4, 121473 +5, 121573 +6, 121625 +7, 121621 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 121857 +1, 122153 +2, 121983 +3, 121983 +4, 121983 +5, 122063 +6, 122133 +7, 122153 +0, 122541 +1, 122837 +2, 122667 +3, 122667 +4, 122667 +5, 122747 +6, 122817 +7, 122837 +0, 123719 +1, 126063 +2, 125893 +3, 125893 +4, 125893 +5, 125973 +6, 126043 +7, 126063 +0, 124107 +1, 126451 +2, 126281 +3, 126281 +4, 126281 +5, 126361 +6, 126431 +7, 126451 +0, 139009 +1, 140705 +2, 140545 +3, 140541 +4, 140541 +5, 138567 +6, 140757 +7, 140705 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 168117 +1, 169669 +2, 170291 +3, 170291 +4, 170291 +5, 169581 +6, 169657 +7, 169669 +0, 778 +1, 778 +2, 778 +3, 778 +4, 778 +5, 1034 +6, 168709 +7, 2058 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fa737515ef44118e6b0539752315584ad6ff5246 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5c9f527c5f52a64637deab41440d1d2acbc58d5d --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log @@ -0,0 +1,1572 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:10:26.251000 192926 torch/distributed/run.py:766] +W0826 16:10:26.251000 192926 torch/distributed/run.py:766] ***************************************** +W0826 16:10:26.251000 192926 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:10:26.251000 192926 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 27 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 4 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank1]:[W826 16:10:44.144516752 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:10:44.144636813 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:10:44.144672243 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:10:44.144715997 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:10:44.144741139 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:10:44.145858996 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:10:44.146372814 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:10:44.288962085 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.837 seconds +time to initialize megatron (seconds): 7.344 +[after megatron is initialized] datetime: 2026-08-26 16:10:50 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.input_layernorm.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:10:50 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:10:50 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (604.27, 611.07) + train/valid/test-data-iterators-setup ..........: (58.09, 67.22) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:10:50 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:11:29] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 39046.5 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092139E+01 | loss scale: 1.0 | grad norm: 30.620 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 + +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 40525.23095703125 | max allocated: 115427.2763671875 | reserved: 117354.0 | max reserved: 117354.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:11:30] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 651.1 | throughput per GPU (TFLOP/s/GPU): 196.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092003E+01 | loss scale: 1.0 | grad norm: 30.566 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:31] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 648.0 | throughput per GPU (TFLOP/s/GPU): 196.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952471E+00 | loss scale: 1.0 | grad norm: 42.237 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:11:33] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1916.0 | throughput per GPU (TFLOP/s/GPU): 66.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476898E+00 | loss scale: 1.0 | grad norm: 7.154 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:33] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 597.8 | throughput per GPU (TFLOP/s/GPU): 213.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.114793E+01 | loss scale: 1.0 | grad norm: 400.648 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:34] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 603.4 | throughput per GPU (TFLOP/s/GPU): 211.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.368878E+00 | loss scale: 1.0 | grad norm: 5.139 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:34] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 602.5 | throughput per GPU (TFLOP/s/GPU): 211.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.975649E+00 | loss scale: 1.0 | grad norm: 3.681 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:35] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 599.9 | throughput per GPU (TFLOP/s/GPU): 212.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785368E+00 | loss scale: 1.0 | grad norm: 5.697 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:36] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 599.2 | throughput per GPU (TFLOP/s/GPU): 212.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.613759E+00 | loss scale: 1.0 | grad norm: 3.002 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:36] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 599.1 | throughput per GPU (TFLOP/s/GPU): 213.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.899922E+00 | loss scale: 1.0 | grad norm: 1155.511 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:37] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 600.1 | throughput per GPU (TFLOP/s/GPU): 212.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.563369E+00 | loss scale: 1.0 | grad norm: 14.007 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:11:37] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 600.6 | throughput per GPU (TFLOP/s/GPU): 212.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.509861E+00 | loss scale: 1.0 | grad norm: 83.626 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:11:37 +[rank0]:[W826 16:11:39.193797993 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c6578dfae1135d63e6eb1d94047b901b5072080c --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv @@ -0,0 +1,96 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 27 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 26289 +1, 26377 +2, 26223 +3, 26223 +4, 26223 +5, 26223 +6, 26355 +7, 26377 +0, 32145 +1, 32231 +2, 32081 +3, 32079 +4, 32079 +5, 32077 +6, 32211 +7, 32231 +0, 36901 +1, 36985 +2, 36835 +3, 36833 +4, 36833 +5, 36831 +6, 36967 +7, 36985 +0, 37785 +1, 37869 +2, 37719 +3, 37717 +4, 37717 +5, 37715 +6, 37851 +7, 37869 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fa737515ef44118e6b0539752315584ad6ff5246 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env @@ -0,0 +1,61 @@ +backend=echo +config=DSV2_LITE_L24 +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=24 +hidden=2048 +ffn_hidden=10944 +moe_ffn_hidden=1408 +num_experts=64 +topk=6 +experts_per_rank=8 +expert_params=8650752 +heads=16 +kv_groups=0 +kv_channels=128 +qk_layernorm=1 +shared_ffn=2816 +shared_expert_overlap=0 +mla=1 +dense_first_layers=1 +moe_layers=23 +rotary_base=10000 +rotary_scaling=40 +qkv_bias=0 +router_balancing=seq_aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=49152 +mean_tokens_per_expert=6144 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b3ff1e6e338fc7d7851f70a8bbd0d381e4b44882 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log @@ -0,0 +1,1535 @@ +============================================================== + DSV2_LITE_L24 / echo (slots/rank=8 policy=matched) + E=64 K=6 H'=1408 m=8 + L=24 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep3/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:11:46.461000 196475 torch/distributed/run.py:766] +W0826 16:11:46.461000 196475 torch/distributed/run.py:766] ***************************************** +W0826 16:11:46.461000 196475 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:11:46.461000 196475 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank [Gloo] Rank 02 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 01 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank [Gloo] Rank 000 is connected to is connected to is connected to 000 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 000 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 16:12:04.735173168 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:12:04.735271225 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:12:04.735309112 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:12:04.735334117 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:12:04.735353593 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:12:04.735563707 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:12:04.735580012 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:12:04.892385656 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.906 seconds +time to initialize megatron (seconds): 6.955 +[after megatron is initialized] datetime: 2026-08-26 16:12:10 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1448: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.input_layernorm.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:12:11 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:12:11 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (652.66, 661.08) + train/valid/test-data-iterators-setup ..........: (60.59, 70.19) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:12:11 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +E0826 16:12:35.310000 196475 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -9) local_rank: 0 (pid: 196562) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 1 (local_rank: 1) + exitcode : -9 (pid: 196563) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196563 +[2]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 2 (local_rank: 2) + exitcode : -9 (pid: 196564) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196564 +[3]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 3 (local_rank: 3) + exitcode : -9 (pid: 196565) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196565 +[4]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 4 (local_rank: 4) + exitcode : -9 (pid: 196566) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196566 +[5]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 5 (local_rank: 5) + exitcode : -9 (pid: 196567) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196567 +[6]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 6 (local_rank: 6) + exitcode : -9 (pid: 196568) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196568 +[7]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 7 (local_rank: 7) + exitcode : -9 (pid: 196569) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196569 +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_16:12:35 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 0 (local_rank: 0) + exitcode : -9 (pid: 196562) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 196562 +============================================================ diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv index c76d61a7d85b4ac2432e123aa75957800b5c9545..d7f7851c59cc32d380b6ac88bfd07764f35688d5 100644 --- a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/mem_samples.csv @@ -78,3 +78,179 @@ 5, 48127 6, 48143 7, 48171 +0, 49929 +1, 49929 +2, 49929 +3, 49929 +4, 49641 +5, 49929 +6, 49929 +7, 49929 +0, 53699 +1, 53699 +2, 53699 +3, 53699 +4, 53411 +5, 53699 +6, 53699 +7, 53699 +0, 54107 +1, 54107 +2, 54107 +3, 54107 +4, 53819 +5, 54107 +6, 54107 +7, 54107 +0, 54385 +1, 54401 +2, 54393 +3, 54409 +4, 54139 +5, 54421 +6, 54413 +7, 54391 +0, 55545 +1, 55545 +2, 55545 +3, 55545 +4, 55257 +5, 55545 +6, 55545 +7, 55545 +0, 56569 +1, 56569 +2, 56569 +3, 56569 +4, 56281 +5, 56569 +6, 56569 +7, 56569 +0, 59801 +1, 59801 +2, 59801 +3, 59801 +4, 59513 +5, 59801 +6, 59801 +7, 59801 +0, 59801 +1, 59801 +2, 59801 +3, 59801 +4, 59513 +5, 59801 +6, 59801 +7, 59801 +0, 105819 +1, 105839 +2, 105881 +3, 105851 +4, 105831 +5, 105837 +6, 105869 +7, 105839 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 110745 +1, 110785 +2, 110785 +3, 110785 +4, 110765 +5, 110785 +6, 110401 +7, 110785 +0, 111393 +1, 111433 +2, 111433 +3, 111433 +4, 111413 +5, 111433 +6, 111049 +7, 111433 +0, 112097 +1, 112121 +2, 112107 +3, 112155 +4, 112119 +5, 112117 +6, 111771 +7, 112135 +0, 130679 +1, 130847 +2, 120531 +3, 131887 +4, 130567 +5, 128851 +6, 129427 +7, 112545 +0, 155643 +1, 155809 +2, 155809 +3, 155425 +4, 155807 +5, 155809 +6, 155809 +7, 155809 +0, 155643 +1, 155809 +2, 155809 +3, 155425 +4, 155807 +5, 155809 +6, 155809 +7, 155809 +0, 154397 +1, 154563 +2, 154563 +3, 154179 +4, 154561 +5, 154563 +6, 154563 +7, 154563 +0, 3363 +1, 3366 +2, 3366 +3, 3366 +4, 3366 +5, 3622 +6, 154561 +7, 4646 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log index d3d45cfdd6dd45d439f002b077ffa97c0f558ab2..5e55b4f1b3e7356d68b7fade96374e3353a7b3fd 100644 --- a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2/train.log @@ -1232,3 +1232,153 @@ done with setup ... training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... [before the start of training step] datetime: 2026-08-26 16:02:44 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:03:22] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38132.5 | throughput per GPU (TFLOP/s/GPU): 4.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.779 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85 + +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81341.77783203125 | max allocated: 96217.232421875 | reserved: 102124.0 | max reserved: 102124.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 16:03:23] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 588.1 | throughput per GPU (TFLOP/s/GPU): 307.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:23] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 644.4 | throughput per GPU (TFLOP/s/GPU): 280.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.694104E+00 | loss scale: 1.0 | grad norm: 66.384 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:03:26] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 2761.9 | throughput per GPU (TFLOP/s/GPU): 65.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338530E+01 | loss scale: 1.0 | grad norm: 19.947 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:26] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 324.9 | throughput per GPU (TFLOP/s/GPU): 556.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.011544E+01 | loss scale: 1.0 | grad norm: 84.156 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:27] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 323.1 | throughput per GPU (TFLOP/s/GPU): 559.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.339844E+01 | loss scale: 1.0 | grad norm: 121.227 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:27] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 329.3 | throughput per GPU (TFLOP/s/GPU): 549.2 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031030E+01 | loss scale: 1.0 | grad norm: 6.141 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:27] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 323.8 | throughput per GPU (TFLOP/s/GPU): 558.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.867838E+00 | loss scale: 1.0 | grad norm: 5.538 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:28] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 323.1 | throughput per GPU (TFLOP/s/GPU): 559.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.620957E+00 | loss scale: 1.0 | grad norm: 5.434 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:28] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 327.5 | throughput per GPU (TFLOP/s/GPU): 552.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.527014E+00 | loss scale: 1.0 | grad norm: 5.273 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:28] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 322.6 | throughput per GPU (TFLOP/s/GPU): 560.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.403372E+00 | loss scale: 1.0 | grad norm: 4.972 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:03:29] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 329.4 | throughput per GPU (TFLOP/s/GPU): 549.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.283264E+00 | loss scale: 1.0 | grad norm: 4.664 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:03:29 +[rank0]:[W826 16:03:30.426110892 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..cd0d23af70af700b3b19ae91d6bb2682f10843a3 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/mem_samples.csv @@ -0,0 +1,248 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 47799 +1, 47799 +2, 47799 +3, 47799 +4, 47511 +5, 47799 +6, 47799 +7, 47799 +0, 49047 +1, 49057 +2, 49049 +3, 49047 +4, 48769 +5, 49053 +6, 49047 +7, 49057 +0, 49929 +1, 49929 +2, 49929 +3, 49929 +4, 49641 +5, 49929 +6, 49929 +7, 49929 +0, 53699 +1, 53699 +2, 53699 +3, 53699 +4, 53411 +5, 53699 +6, 53699 +7, 53699 +0, 54107 +1, 54107 +2, 54107 +3, 54107 +4, 53819 +5, 54107 +6, 54107 +7, 54107 +0, 54433 +1, 54419 +2, 54425 +3, 54437 +4, 54117 +5, 54429 +6, 54425 +7, 54411 +0, 55545 +1, 55545 +2, 55545 +3, 55545 +4, 55257 +5, 55545 +6, 55545 +7, 55545 +0, 56569 +1, 56569 +2, 56569 +3, 56569 +4, 56281 +5, 56569 +6, 56569 +7, 56569 +0, 59801 +1, 59801 +2, 59801 +3, 59801 +4, 59513 +5, 59801 +6, 59801 +7, 59801 +0, 59801 +1, 59801 +2, 59801 +3, 59801 +4, 59513 +5, 59801 +6, 59801 +7, 59801 +0, 105771 +1, 105791 +2, 105783 +3, 105773 +4, 105757 +5, 105781 +6, 105775 +7, 105785 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 106307 +1, 106347 +2, 106347 +3, 106347 +4, 106327 +5, 106347 +6, 106347 +7, 106347 +0, 110843 +1, 110887 +2, 110893 +3, 110931 +4, 110877 +5, 110919 +6, 110897 +7, 110885 +0, 111783 +1, 111823 +2, 111823 +3, 111823 +4, 111803 +5, 111823 +6, 111823 +7, 111823 +0, 142195 +1, 142307 +2, 142307 +3, 142307 +4, 142335 +5, 142307 +6, 142307 +7, 142307 +0, 143677 +1, 141263 +2, 155809 +3, 143799 +4, 142513 +5, 143917 +6, 144203 +7, 144043 +0, 155259 +1, 155809 +2, 155809 +3, 155809 +4, 155807 +5, 155809 +6, 155809 +7, 155809 +0, 155259 +1, 155809 +2, 155809 +3, 155809 +4, 155807 +5, 155809 +6, 155809 +7, 155809 +0, 3366 +1, 3366 +2, 3366 +3, 3366 +4, 3622 +5, 154561 +6, 4646 +7, 3366 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env new file mode 100644 index 0000000000000000000000000000000000000000..dac66d76a70876595c374526a60455b1044c19d9 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log new file mode 100644 index 0000000000000000000000000000000000000000..562b787cc89fae87af4f71efc988c8aaf3549513 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/train.log @@ -0,0 +1,1384 @@ +[16:03:38] WARN: bracing-regions-through-kernels lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:03:40.105000 175558 torch/distributed/run.py:766] +W0826 16:03:40.105000 175558 torch/distributed/run.py:766] ***************************************** +W0826 16:03:40.105000 175558 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:03:40.105000 175558 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 1 is connected to 27 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank 34 is connected to is connected to 577 is connected to peer ranks. peer ranks. 7Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. 77Expected number of connected peer ranks is : + +7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 5 is connected to 07 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7[Gloo] Rank + +1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to [Gloo] Rank 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 00 + is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 16:03:58.786294961 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:03:58.786399157 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:03:58.786422234 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:03:58.786441026 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:03:58.786715267 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:03:58.788307701 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:03:58.788328101 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:03:58.937972899 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.658 seconds +time to initialize megatron (seconds): 6.618 +[after megatron is initialized] datetime: 2026-08-26 16:04:04 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.k_layernorm.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:04:05 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:04:05 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (836.48, 854.02) + train/valid/test-data-iterators-setup ..........: (64.97, 70.55) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:04:05 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:04:42] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 37374.4 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115992E+01 | loss scale: 1.0 | grad norm: 44.778 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 81341.77783203125 | max allocated: 96217.232421875 | reserved: 102124.0 | max reserved: 102124.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 16:04:43] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 575.0 | throughput per GPU (TFLOP/s/GPU): 314.5 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117604E+01 | loss scale: 1.0 | grad norm: 44.306 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:44] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 671.9 | throughput per GPU (TFLOP/s/GPU): 269.1 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.693775E+00 | loss scale: 1.0 | grad norm: 66.340 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:04:46] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 2690.8 | throughput per GPU (TFLOP/s/GPU): 67.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338493E+01 | loss scale: 1.0 | grad norm: 19.961 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:47] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 325.1 | throughput per GPU (TFLOP/s/GPU): 556.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.010851E+01 | loss scale: 1.0 | grad norm: 84.114 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:47] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 322.7 | throughput per GPU (TFLOP/s/GPU): 560.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.350914E+01 | loss scale: 1.0 | grad norm: 121.505 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:47] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 324.3 | throughput per GPU (TFLOP/s/GPU): 557.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.031437E+01 | loss scale: 1.0 | grad norm: 6.222 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:48] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 323.0 | throughput per GPU (TFLOP/s/GPU): 559.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.877584E+00 | loss scale: 1.0 | grad norm: 5.584 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:48] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 323.3 | throughput per GPU (TFLOP/s/GPU): 559.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.632164E+00 | loss scale: 1.0 | grad norm: 5.487 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:48] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 323.5 | throughput per GPU (TFLOP/s/GPU): 558.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.538463E+00 | loss scale: 1.0 | grad norm: 5.323 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:49] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 325.9 | throughput per GPU (TFLOP/s/GPU): 554.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.413701E+00 | loss scale: 1.0 | grad norm: 5.059 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:04:49] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 322.5 | throughput per GPU (TFLOP/s/GPU): 560.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.292517E+00 | loss scale: 1.0 | grad norm: 4.741 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:04:49 +[rank0]:[W826 16:04:50.642132270 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..d5d693f1bc16ca64052c742cf0ad1463048be66f --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 7637 +1, 7657 +2, 7357 +3, 7557 +4, 7697 +5, 7757 +6, 7757 +7, 7757 +0, 44781 +1, 44843 +2, 44869 +3, 44803 +4, 44823 +5, 44845 +6, 44827 +7, 44827 +0, 45509 +1, 45511 +2, 45521 +3, 45483 +4, 45507 +5, 45521 +6, 45511 +7, 45521 +0, 46919 +1, 46921 +2, 46931 +3, 46893 +4, 46917 +5, 46931 +6, 46921 +7, 46931 +0, 47015 +1, 47017 +2, 47027 +3, 46989 +4, 47013 +5, 47027 +6, 47017 +7, 47027 +0, 47073 +1, 47079 +2, 47093 +3, 47061 +4, 47083 +5, 47101 +6, 47085 +7, 47099 +0, 47879 +1, 47881 +2, 47891 +3, 47853 +4, 47877 +5, 47891 +6, 47881 +7, 47891 +0, 48135 +1, 48137 +2, 48147 +3, 48109 +4, 48133 +5, 48147 +6, 48137 +7, 48147 +0, 48779 +1, 48801 +2, 48795 +3, 48773 +4, 48777 +5, 48795 +6, 48765 +7, 48795 +0, 49133 +1, 49153 +2, 49147 +3, 49127 +4, 49129 +5, 49147 +6, 49119 +7, 49147 +0, 100837 +1, 100859 +2, 100843 +3, 100859 +4, 100839 +5, 100845 +6, 100847 +7, 100835 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 102061 +1, 102849 +2, 102067 +3, 102075 +4, 102077 +5, 102259 +6, 102259 +7, 102259 +0, 102997 +1, 103497 +2, 102715 +3, 102915 +4, 102917 +5, 103099 +6, 102907 +7, 103003 +0, 103687 +1, 104149 +2, 103367 +3, 103593 +4, 103597 +5, 103737 +6, 103587 +7, 103659 +0, 133045 +1, 133533 +2, 132963 +3, 133047 +4, 132993 +5, 133251 +6, 132943 +7, 133155 +0, 140013 +1, 140011 +2, 139051 +3, 140219 +4, 140203 +5, 139617 +6, 147795 +7, 140337 +0, 151641 +1, 151639 +2, 151641 +3, 151651 +4, 151635 +5, 151641 +6, 151643 +7, 151641 +0, 151641 +1, 151639 +2, 151641 +3, 151651 +4, 151635 +5, 151641 +6, 151643 +7, 151641 +0, 150825 +1, 150823 +2, 150825 +3, 150835 +4, 150819 +5, 150825 +6, 150827 +7, 150825 +0, 2186 +1, 906 +2, 906 +3, 906 +4, 906 +5, 906 +6, 1162 +7, 150825 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fa212f5afb663acbb8be5e69bb0c696cdeca4f05 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..219be1e3f6320f012d752295f7f735658054bcec --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/train.log @@ -0,0 +1,2838 @@ +[16:04:59] WARN: bracing-regions-through-kernels lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:05:00.397000 179025 torch/distributed/run.py:766] +W0826 16:05:00.397000 179025 torch/distributed/run.py:766] ***************************************** +W0826 16:05:00.397000 179025 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:05:00.397000 179025 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4[Gloo] Rank [Gloo] Rank [Gloo] Rank is connected to 31[Gloo] Rank 2756 is connected to is connected to [Gloo] Rank is connected to peer ranks. is connected to is connected to 7777Expected number of connected peer ranks is : 77 peer ranks. peer ranks. peer ranks. is connected to 7 peer ranks. + peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 peer ranks. +77 + + +Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 27 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 0000 is connected to is connected to is connected to is connected to 0000 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 +00 + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank3]:[W826 16:05:18.669807286 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:05:18.669817227 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:05:18.669817945 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:05:18.669865816 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:05:18.669925521 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:05:18.669959595 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:05:18.669974297 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:05:18.818106384 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.777 seconds +time to initialize megatron (seconds): 7.136 +[after megatron is initialized] datetime: 2026-08-26 16:05:24 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_proj.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.41.pre_mlp_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.pre_mlp_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.k_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.router.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:05:25 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:05:26 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (1206.86, 1450.48) + train/valid/test-data-iterators-setup ..........: (64.30, 71.03) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:05:26 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:06:04] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38043.2 | throughput per GPU (TFLOP/s/GPU): 2.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095255E+01 | loss scale: 1.0 | grad norm: 54.341 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 80701.00390625 | max allocated: 95238.11279296875 | reserved: 96618.0 | max reserved: 96618.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 16:06:05] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1046.0 | throughput per GPU (TFLOP/s/GPU): 85.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095489E+01 | loss scale: 1.0 | grad norm: 56.501 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:06] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1070.5 | throughput per GPU (TFLOP/s/GPU): 83.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008672E+01 | loss scale: 1.0 | grad norm: 180.236 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:06:09] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 3116.4 | throughput per GPU (TFLOP/s/GPU): 28.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.603939E+00 | loss scale: 1.0 | grad norm: 8.752 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:09] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 405.1 | throughput per GPU (TFLOP/s/GPU): 220.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309446E+00 | loss scale: 1.0 | grad norm: 4.787 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:10] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 401.6 | throughput per GPU (TFLOP/s/GPU): 221.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.226131E+00 | loss scale: 1.0 | grad norm: 5.010 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:10] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 400.8 | throughput per GPU (TFLOP/s/GPU): 222.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.063538E+00 | loss scale: 1.0 | grad norm: 3.370 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:10] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 399.8 | throughput per GPU (TFLOP/s/GPU): 222.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.875765E+00 | loss scale: 1.0 | grad norm: 3.336 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:11] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 399.6 | throughput per GPU (TFLOP/s/GPU): 223.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.813617E+00 | loss scale: 1.0 | grad norm: 3.140 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:11] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 397.8 | throughput per GPU (TFLOP/s/GPU): 224.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.773812E+00 | loss scale: 1.0 | grad norm: 3.152 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:12] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 397.0 | throughput per GPU (TFLOP/s/GPU): 224.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.764331E+00 | loss scale: 1.0 | grad norm: 3.004 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:06:12] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 396.5 | throughput per GPU (TFLOP/s/GPU): 224.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.736594E+00 | loss scale: 1.0 | grad norm: 2.873 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:06:12 +[rank0]:[W826 16:06:13.897114691 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 16:06:19.077000 179025 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:06:19.078000 179025 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 179122 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 179025 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..13cea95dab927ab78f17e1058c7c6018ea28a98e --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/mem_samples.csv @@ -0,0 +1,264 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 907 +1, 879 +2, 875 +3, 865 +4, 871 +5, 879 +6, 909 +7, 873 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 39169 +1, 38995 +2, 39197 +3, 38995 +4, 38995 +5, 39197 +6, 38995 +7, 39197 +0, 44991 +1, 44993 +2, 45003 +3, 44965 +4, 44989 +5, 45003 +6, 44993 +7, 45003 +0, 45897 +1, 45899 +2, 45909 +3, 45871 +4, 45895 +5, 45909 +6, 45899 +7, 45909 +0, 46919 +1, 46921 +2, 46931 +3, 46893 +4, 46917 +5, 46931 +6, 46921 +7, 46931 +0, 47015 +1, 47017 +2, 47027 +3, 46989 +4, 47013 +5, 47027 +6, 47017 +7, 47027 +0, 47221 +1, 47247 +2, 47233 +3, 47199 +4, 47221 +5, 47229 +6, 47201 +7, 47219 +0, 47879 +1, 47881 +2, 47891 +3, 47853 +4, 47877 +5, 47891 +6, 47881 +7, 47891 +0, 48135 +1, 48137 +2, 48147 +3, 48109 +4, 48133 +5, 48147 +6, 48137 +7, 48147 +0, 49133 +1, 49153 +2, 49147 +3, 49127 +4, 49129 +5, 49147 +6, 49119 +7, 49147 +0, 49133 +1, 49153 +2, 49147 +3, 49127 +4, 49129 +5, 49147 +6, 49119 +7, 49147 +0, 100817 +1, 100865 +2, 100819 +3, 100825 +4, 100823 +5, 100819 +6, 100833 +7, 100831 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 102253 +1, 102081 +2, 102067 +3, 102075 +4, 102077 +5, 102067 +6, 102067 +7, 102067 +0, 103477 +1, 103209 +2, 102715 +3, 103299 +4, 103301 +5, 103291 +6, 102907 +7, 103291 +0, 104149 +1, 103919 +2, 103385 +3, 103969 +4, 103969 +5, 103961 +6, 103597 +7, 103951 +0, 133525 +1, 133533 +2, 132771 +3, 133527 +4, 133569 +5, 133539 +6, 133519 +7, 133539 +0, 139125 +1, 143003 +2, 132365 +3, 134511 +4, 132503 +5, 133587 +6, 137973 +7, 134455 +0, 151737 +1, 151735 +2, 151641 +3, 151747 +4, 151731 +5, 151737 +6, 151643 +7, 151641 +0, 151737 +1, 151735 +2, 151641 +3, 151747 +4, 151731 +5, 151737 +6, 151643 +7, 151641 +0, 150921 +1, 150919 +2, 150825 +3, 150931 +4, 150915 +5, 150921 +6, 150827 +7, 150825 +0, 906 +1, 906 +2, 906 +3, 906 +4, 1162 +5, 150921 +6, 2186 +7, 906 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fa212f5afb663acbb8be5e69bb0c696cdeca4f05 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e11ce9ec882281bc0e1648718b471310ca597f78 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/train.log @@ -0,0 +1,2838 @@ +[16:06:20] WARN: bracing-regions-through-kernels lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:06:21.577000 182499 torch/distributed/run.py:766] +W0826 16:06:21.577000 182499 torch/distributed/run.py:766] ***************************************** +W0826 16:06:21.577000 182499 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:06:21.577000 182499 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 163257 is connected to is connected to is connected to is connected to is connected to is connected to 777777 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777777 + + + + + +4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to [Gloo] Rank 00 peer ranks. is connected to [Gloo] Rank Expected number of connected peer ranks is : 000 peer ranks. is connected to +Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W826 16:06:39.014241383 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:06:39.014386969 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 16:06:39.014484283 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:06:39.014481616 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:06:39.014485337 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:06:39.014485958 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:06:39.018958773 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:06:40.186230862 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.764 seconds +time to initialize megatron (seconds): 6.793 +[after megatron is initialized] datetime: 2026-08-26 16:06:45 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_proj.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.k_layernorm.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.router.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:06:47 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:06:47 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (1318.75, 1378.13) + train/valid/test-data-iterators-setup ..........: (61.68, 67.57) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 16:06:47 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:07:25] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38024.0 | throughput per GPU (TFLOP/s/GPU): 2.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095255E+01 | loss scale: 1.0 | grad norm: 54.333 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 80701.00390625 | max allocated: 95238.11279296875 | reserved: 97098.0 | max reserved: 97098.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 16:07:26] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1125.7 | throughput per GPU (TFLOP/s/GPU): 79.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095489E+01 | loss scale: 1.0 | grad norm: 56.497 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:27] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1093.9 | throughput per GPU (TFLOP/s/GPU): 81.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008450E+01 | loss scale: 1.0 | grad norm: 179.129 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:07:30] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 3172.6 | throughput per GPU (TFLOP/s/GPU): 28.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.603470E+00 | loss scale: 1.0 | grad norm: 8.794 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:31] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 405.7 | throughput per GPU (TFLOP/s/GPU): 219.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309288E+00 | loss scale: 1.0 | grad norm: 4.161 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:31] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 405.1 | throughput per GPU (TFLOP/s/GPU): 220.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.278572E+00 | loss scale: 1.0 | grad norm: 11.166 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:32] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 400.8 | throughput per GPU (TFLOP/s/GPU): 222.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.104971E+00 | loss scale: 1.0 | grad norm: 3.463 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:32] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 400.1 | throughput per GPU (TFLOP/s/GPU): 222.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.914668E+00 | loss scale: 1.0 | grad norm: 3.126 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:32] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 406.4 | throughput per GPU (TFLOP/s/GPU): 219.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.859293E+00 | loss scale: 1.0 | grad norm: 3.016 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:33] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 398.9 | throughput per GPU (TFLOP/s/GPU): 223.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.816747E+00 | loss scale: 1.0 | grad norm: 3.041 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:33] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 397.6 | throughput per GPU (TFLOP/s/GPU): 224.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.805655E+00 | loss scale: 1.0 | grad norm: 2.922 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:07:34] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 397.1 | throughput per GPU (TFLOP/s/GPU): 224.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.776004E+00 | loss scale: 1.0 | grad norm: 2.870 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:07:34 +[rank0]:[W826 16:07:35.381288864 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 16:07:40.273000 182499 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:07:40.274000 182499 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 182594 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 182499 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/mem_samples.csv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..c48d300ffd5bd74548e58b99f73952d65f15d7c6 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/mem_samples.csv @@ -0,0 +1,256 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 2 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 629 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1831 +1, 1831 +2, 1831 +3, 1831 +4, 1831 +5, 1831 +6, 1831 +7, 1831 +0, 38993 +1, 38993 +2, 38993 +3, 38993 +4, 38993 +5, 38993 +6, 38993 +7, 38993 +0, 44939 +1, 44919 +2, 44951 +3, 44929 +4, 44937 +5, 44947 +6, 44957 +7, 44967 +0, 45453 +1, 45479 +2, 45477 +3, 45435 +4, 45429 +5, 45455 +6, 45439 +7, 45445 +0, 46919 +1, 46921 +2, 46931 +3, 46893 +4, 46917 +5, 46931 +6, 46921 +7, 46931 +0, 47015 +1, 47017 +2, 47027 +3, 46989 +4, 47013 +5, 47027 +6, 47017 +7, 47027 +0, 47213 +1, 47215 +2, 47241 +3, 47197 +4, 47209 +5, 47221 +6, 47213 +7, 47237 +0, 47879 +1, 47881 +2, 47891 +3, 47853 +4, 47877 +5, 47891 +6, 47881 +7, 47891 +0, 48135 +1, 48137 +2, 48147 +3, 48109 +4, 48133 +5, 48147 +6, 48137 +7, 48147 +0, 49133 +1, 49153 +2, 49147 +3, 49127 +4, 49129 +5, 49147 +6, 49119 +7, 49147 +0, 49133 +1, 49153 +2, 49147 +3, 49127 +4, 49129 +5, 49147 +6, 49119 +7, 49147 +0, 100859 +1, 100887 +2, 100871 +3, 100851 +4, 100845 +5, 100851 +6, 100907 +7, 100865 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 101347 +1, 101367 +2, 101353 +3, 101361 +4, 101363 +5, 101353 +6, 101353 +7, 101353 +0, 102061 +1, 102273 +2, 102067 +3, 102363 +4, 102269 +5, 102835 +6, 102259 +7, 102355 +0, 102709 +1, 102921 +2, 102715 +3, 103011 +4, 102917 +5, 103483 +6, 102907 +7, 103003 +0, 103369 +1, 103575 +2, 103343 +3, 103669 +4, 103559 +5, 104131 +6, 103557 +7, 103643 +0, 133045 +1, 133053 +2, 133251 +3, 133527 +4, 133377 +5, 133539 +6, 132943 +7, 133059 +0, 141287 +1, 137011 +2, 139149 +3, 142357 +4, 139145 +5, 140113 +6, 147219 +7, 146353 +0, 151641 +1, 151639 +2, 151641 +3, 151747 +4, 151635 +5, 151641 +6, 151643 +7, 151641 +0, 151641 +1, 151639 +2, 151641 +3, 151747 +4, 151635 +5, 151641 +6, 151643 +7, 151641 +0, 150825 +1, 150823 +2, 150825 +3, 150931 +4, 150819 +5, 150825 +6, 150827 +7, 150825 +0, 905 +1, 1162 +2, 150825 +3, 2186 +4, 906 +5, 906 +6, 906 +7, 906 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/run.env b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/run.env new file mode 100644 index 0000000000000000000000000000000000000000..fa212f5afb663acbb8be5e69bb0c696cdeca4f05 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/run.env @@ -0,0 +1,61 @@ +backend=echo +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=bracing-regions-through-kernels +commit=9a236793 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=static +capacity=2.00 +cuda_graph=1 +probe_mode=0 +async_grad_reduce=0 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/train.log b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e2a77b32373d35fec8afa82b04070a470caf8037 --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/train.log @@ -0,0 +1,2838 @@ +[16:07:42] WARN: bracing-regions-through-kernels lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / echo (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=2.00 cg=1 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 9a236793 +============================================================== +W0826 16:07:43.773000 185950 torch/distributed/run.py:766] +W0826 16:07:43.773000 185950 torch/distributed/run.py:766] ***************************************** +W0826 16:07:43.773000 185950 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 16:07:43.773000 185950 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 48 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 6144 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 768 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 128 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 48 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 1 is connected to 37 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 16:08:02.079090605 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 16:08:02.079097102 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 16:08:02.079211463 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 16:08:02.082625154 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 16:08:02.082659698 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 16:08:02.084272501 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 16:08:02.087605084 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 16:08:02.226513977 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.155 seconds +time to initialize megatron (seconds): 7.597 +[after megatron is initialized] datetime: 2026-08-26 16:08:08 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4748687360 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 21 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (49033984 elements, 49033984 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.mlp.router.weight + module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.45.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.45.mlp.router.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.weight +Params for bucket 3 (46672384 elements, 46672384 padded size): + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight +Params for bucket 4 (49031936 elements, 49031936 padded size): + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.42.self_attention.linear_proj.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.linear_qkv.weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.42.mlp.router.weight + module.decoder.layers.42.self_attention.k_layernorm.weight +Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight + module.decoder.layers.39.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.q_layernorm.weight + module.decoder.layers.39.self_attention.linear_qkv.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight +Params for bucket 6 (49031936 elements, 49031936 padded size): + module.decoder.layers.35.mlp.router.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.37.pre_mlp_layernorm.weight + module.decoder.layers.36.mlp.router.weight + module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight + module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.37.self_attention.linear_proj.weight + module.decoder.layers.36.self_attention.linear_qkv.weight +Params for bucket 7 (46672384 elements, 46672384 padded size): + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight +Params for bucket 8 (49031936 elements, 49031936 padded size): + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.weight + module.decoder.layers.31.mlp.router.weight + module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.30.self_attention.q_layernorm.weight + module.decoder.layers.32.mlp.router.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight +Params for bucket 9 (46672384 elements, 46672384 padded size): + module.decoder.layers.29.mlp.router.weight + module.decoder.layers.29.self_attention.k_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.self_attention.q_layernorm.weight + module.decoder.layers.29.self_attention.linear_qkv.weight + module.decoder.layers.29.self_attention.linear_proj.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight +Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.26.mlp.router.weight + module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.25.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight + module.decoder.layers.27.mlp.router.weight + module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.q_layernorm.weight + module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.27.self_attention.q_layernorm.weight +Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.q_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.k_layernorm.weight + module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.23.self_attention.linear_proj.weight +Params for bucket 12 (49031936 elements, 49031936 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.22.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.self_attention.q_layernorm.weight +Params for bucket 13 (46672384 elements, 46672384 padded size): + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.self_attention.k_layernorm.weight + module.decoder.layers.18.self_attention.q_layernorm.weight +Params for bucket 14 (49031936 elements, 49031936 padded size): + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.pre_mlp_layernorm.weight +Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.k_layernorm.weight + module.decoder.layers.13.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 16 (49031936 elements, 49031936 padded size): + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight +Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 18 (49031936 elements, 49031936 padded size): + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.weight +Params for bucket 19 (46672384 elements, 46672384 padded size): + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.0.mlp.router.weight +Params for bucket 21 (111413248 elements, 111413248 padded size): + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 +Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc2.weight8 + module.decoder.layers.47.mlp.experts.linear_fc2.weight0 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight13 + module.decoder.layers.47.mlp.experts.linear_fc2.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 +Params for bucket 2 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight5 + module.decoder.layers.47.mlp.experts.linear_fc1.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc2.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.46.mlp.experts.linear_fc2.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 +Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight13 + module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 +Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.44.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 +Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight9 + module.decoder.layers.43.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 +Params for bucket 9 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 +Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.42.mlp.experts.linear_fc2.weight11 + module.decoder.layers.42.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight10 + module.decoder.layers.42.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc1.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight11 + module.decoder.layers.42.mlp.experts.linear_fc1.weight9 + module.decoder.layers.42.mlp.experts.linear_fc1.weight7 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 + module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight8 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight2 +Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight13 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight9 + module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.41.mlp.experts.linear_fc1.weight12 + module.decoder.layers.41.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc2.weight6 +Params for bucket 13 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc1.weight10 + module.decoder.layers.41.mlp.experts.linear_fc1.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight6 + module.decoder.layers.41.mlp.experts.linear_fc1.weight4 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 +Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight5 + module.decoder.layers.40.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc2.weight1 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 +Params for bucket 15 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 +Params for bucket 16 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 +Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc1.weight5 + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 +Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc1.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 +Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc1.weight9 + module.decoder.layers.37.mlp.experts.linear_fc1.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight11 + module.decoder.layers.36.mlp.experts.linear_fc2.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 +Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight11 + module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight9 + module.decoder.layers.36.mlp.experts.linear_fc1.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 + module.decoder.layers.36.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 +Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc2.weight10 + module.decoder.layers.35.mlp.experts.linear_fc2.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.35.mlp.experts.linear_fc2.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight3 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 +Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 +Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight3 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 +Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 +Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc2.weight15 + module.decoder.layers.32.mlp.experts.linear_fc2.weight13 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc2.weight9 +Params for bucket 29 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight8 +Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.31.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 +Params for bucket 31 (40894464 elements, 40894464 padded size): + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 + module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 +Params for bucket 32 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 + module.decoder.layers.30.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight6 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.30.mlp.experts.linear_fc2.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.30.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 +Params for bucket 33 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc1.weight10 + module.decoder.layers.30.mlp.experts.linear_fc1.weight8 + module.decoder.layers.30.mlp.experts.linear_fc1.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 + module.decoder.layers.30.mlp.experts.linear_fc1.weight9 + module.decoder.layers.30.mlp.experts.linear_fc1.weight7 + module.decoder.layers.30.mlp.experts.linear_fc1.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 +Params for bucket 34 (40894464 elements, 40894464 padded size): + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight11 + module.decoder.layers.29.mlp.experts.linear_fc2.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight8 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight15 +Params for bucket 35 (40894464 elements, 40894464 padded size): + module.decoder.layers.29.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight8 + module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 + module.decoder.layers.29.mlp.experts.linear_fc1.weight2 + module.decoder.layers.29.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc1.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight11 + module.decoder.layers.29.mlp.experts.linear_fc1.weight7 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 + module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 +Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight9 + module.decoder.layers.28.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight6 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight5 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 +Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight9 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight14 +Params for bucket 38 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight13 + module.decoder.layers.27.mlp.experts.linear_fc2.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 +Params for bucket 39 (40894464 elements, 40894464 padded size): + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 +Params for bucket 41 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight3 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 +Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight6 +Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 +Params for bucket 44 (40894464 elements, 40894464 padded size): + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 +Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight9 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 +Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 +Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 +Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight12 + module.decoder.layers.22.mlp.experts.linear_fc1.weight10 + module.decoder.layers.22.mlp.experts.linear_fc1.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 +Params for bucket 49 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc2.weight9 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight8 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 + module.decoder.layers.21.mlp.experts.linear_fc2.weight12 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 +Params for bucket 50 (40894464 elements, 40894464 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight10 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 +Params for bucket 51 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight13 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight15 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 +Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 +Params for bucket 53 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight12 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.19.mlp.experts.linear_fc1.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 +Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight8 +Params for bucket 55 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc1.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 +Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 +Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 + module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight10 + module.decoder.layers.17.mlp.experts.linear_fc1.weight8 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 +Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight12 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 +Params for bucket 59 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 +Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 61 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 +Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 +Params for bucket 63 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 +Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 +Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 66 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 67 (40894464 elements, 40894464 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 +Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 +Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 74 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 +Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 78 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 79 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 83 (40894464 elements, 40894464 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 +Params for bucket 85 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 +Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 +Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 +Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 +Params for bucket 89 (25165824 elements, 25165824 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:08:09 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 16:08:09 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (1119.78, 1299.06) + train/valid/test-data-iterators-setup ..........: (62.13, 70.26) +[before the start of training step] datetime: 2026-08-26 16:08:09 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") + [2026-08-26 16:08:47] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 38244.6 | throughput per GPU (TFLOP/s/GPU): 2.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095255E+01 | loss scale: 1.0 | grad norm: 54.343 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.90 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 30.10 +Number of parameters in most loaded shard in billions: 30.1034 +Theoretical memory footprints: weight and optimizer=215316.59 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 80701.00390625 | max allocated: 95238.11279296875 | reserved: 96330.0 | max reserved: 96330.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") + [2026-08-26 16:08:49] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 1045.9 | throughput per GPU (TFLOP/s/GPU): 85.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095489E+01 | loss scale: 1.0 | grad norm: 56.504 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:50] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 1071.2 | throughput per GPU (TFLOP/s/GPU): 83.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008411E+01 | loss scale: 1.0 | grad norm: 178.917 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-26 16:08:53] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 3152.6 | throughput per GPU (TFLOP/s/GPU): 28.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.603559E+00 | loss scale: 1.0 | grad norm: 8.784 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:53] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 407.6 | throughput per GPU (TFLOP/s/GPU): 218.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.309261E+00 | loss scale: 1.0 | grad norm: 8.491 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:54] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 402.7 | throughput per GPU (TFLOP/s/GPU): 221.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.124468E+00 | loss scale: 1.0 | grad norm: 2.877 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:54] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 401.2 | throughput per GPU (TFLOP/s/GPU): 222.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.976077E+00 | loss scale: 1.0 | grad norm: 2.847 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:54] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 400.8 | throughput per GPU (TFLOP/s/GPU): 222.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.835667E+00 | loss scale: 1.0 | grad norm: 2.895 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:55] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 401.4 | throughput per GPU (TFLOP/s/GPU): 222.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.793233E+00 | loss scale: 1.0 | grad norm: 2.665 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:55] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 397.8 | throughput per GPU (TFLOP/s/GPU): 224.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.753614E+00 | loss scale: 1.0 | grad norm: 2.652 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:56] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 400.1 | throughput per GPU (TFLOP/s/GPU): 222.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.747486E+00 | loss scale: 1.0 | grad norm: 2.530 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 16:08:56] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 397.1 | throughput per GPU (TFLOP/s/GPU): 224.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.724425E+00 | loss scale: 1.0 | grad norm: 2.438 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 16:08:56 +[rank0]:[W826 16:08:57.712445829 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 16:09:02.472000 185950 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 16:09:02.472000 185950 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 186042 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 185950 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/results.tsv b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/results.tsv index 23883b972a9d9926c9ff2115972b1197f7855f33..bc1f933bf7078c4545561d1e6c5a40fa484cce53 100644 --- a/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/results.tsv +++ b/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/results.tsv @@ -1,2 +1,9 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir QWEN3_235B echo matched 16 8192 8 2.00 none 1 OK 323.26 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN3_235B echo matched 16 8192 8 2.00 none 2 OK 324.40 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep2 +QWEN3_235B echo matched 16 8192 8 2.00 none 3 OK 324.13 322.10 155809 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3 +QWEN3_30B echo matched 16 4096 8 2.00 none 1 OK 399.01 396.50 151651 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN3_30B echo matched 16 4096 8 2.00 none 2 OK 399.94 396.50 151747 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep2 +QWEN3_30B echo matched 16 4096 8 2.00 none 3 OK 400.01 396.50 151747 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3 +DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 600.57 598.10 170291 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 +DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 2 OK 600.63 598.10 170291 /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep2 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt index 1f856c93f57f8afbd7a6209003a0a976977bbdc1..9e3213ec55b1dd4529c68355d61489c440825f6f 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +++ b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt @@ -1,7 +1,7 @@ ============================================================== PROVENANCE: echo__tierbprobe__gbs8 ============================================================== -captured_at : 2026-08-26T15:32:33+00:00 +captured_at : 2026-08-26T16:16:43+00:00 hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 kernel : 6.8.0-1043-nvidia cwd : /workspace/blackwell_run_scripts @@ -27,64 +27,3 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: - GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID - GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A - GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A - GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A - GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A - GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A - GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A - GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A - GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A - NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE - NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE - NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS - NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS - NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS - unavailable - ----- toolchain ----------------------------------------------- - nvcc : release 12.9, V12.9.86 - gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 - python : 3.12.3 - torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 - torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] - transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] - deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] - ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] - moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] - megatron : 0.14.0rc5 [None] - flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] - numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] - huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] - ----- repositories -------------------------------------------- - Megatron-MoE-Echo token-drop-analysis b099fd8494b0 - Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 - UltraEP main 94cab099b44f - DeepEP HEAD 34152ae28f80 (+3 untracked: build output) - MoonEP megatron-integration c27d35da15f2 - blackwell_run_scripts granularity-echo d11391fc9103 - ----- environment --------------------------------------------- - CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 - CUDA_DRIVER_VERSION=575.57.08 - CUDA_HOME=/usr/local/cuda - CUDA_MODULE_LOADING=LAZY - CUDA_VERSION=12.9.1.010 - HF_REMOTE_ROOT=tierb_study_20260826_144644 - NCCL_VERSION=2.27.3 - NVSHMEM_VERSION=3.2.5 - PYTORCH_BUILD_NUMBER=0 - PYTORCH_BUILD_VERSION=2.8.0a0+5228986 - PYTORCH_HOME=/opt/pytorch/pytorch - PYTORCH_VERSION=2.8.0a0+5228986 - TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 - TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX - TORCH_NCCL_USE_COMM_NONBLOCKING=0 - ----- shared library resolution ------------------------------- - libnvshmem_host.so: not in ldconfig - libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 - lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 - -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tierb_study_20260826_144644/hf_watcher.log b/tierb_study_20260826_144644/hf_watcher.log index 3edb44d22b5899c33b2eb0255f9321f046d41ecc..9332d43b177592a8dab4f3e60b98cadbe38a6ad5 100644 --- a/tierb_study_20260826_144644/hf_watcher.log +++ b/tierb_study_20260826_144644/hf_watcher.log @@ -26,3 +26,8 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [2026-08-26T16:02:44+00:00] backup pass + uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +Already logged in to Hugging Face as: AbhishekGhosh1998 +Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) +[2026-08-26T16:16:44+00:00] backup pass diff --git a/tierb_study_20260826_144644/hf_watcher.pid b/tierb_study_20260826_144644/hf_watcher.pid index fc665fbeaa06937a8aa2dcc02d1a073bbe4882fc..cd2c9878dc7ce62935f32be364d724ff6f386145 100644 --- a/tierb_study_20260826_144644/hf_watcher.pid +++ b/tierb_study_20260826_144644/hf_watcher.pid @@ -1 +1 @@ -99772 +203270 diff --git a/tierb_study_20260826_144644/study.log b/tierb_study_20260826_144644/study.log index 144b3a66b725683a22dee6a641cac98c84d459ee..acc359b0a999f9fd82ed3414640697ad4b81ddfb 100644 --- a/tierb_study_20260826_144644/study.log +++ b/tierb_study_20260826_144644/study.log @@ -853,3 +853,36 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [16:00:58] CELL QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00 (slots=16) [guard] iter 8/12 (70s) [guard] iter 12/12 (80s) + [guard] iter 9/12 (70s) + [guard] iter 12/12 (80s) + [guard] iter 10/12 (70s) + [guard] iter 12/12 (80s) +-------------------------------------------------------------------------- +[16:04:58] CELL QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00 (slots=16) + [guard] iter 3/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) + [guard] iter 3/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) + [guard] iter 3/12 (70s) + [guard] iter 12/12 (80s) + [guard] tearing down process group (outcome=DONE) +-------------------------------------------------------------------------- +[16:09:04] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00 (slots=8) + [guard] iter 7/12 (70s) + [guard] iter 12/12 (80s) + [guard] iter 7/12 (70s) + [guard] iter 12/12 (80s) +study_root: /workspace/tierb_study_20260826_144644 +gbs: 8 +stages: 7,4,8,9 +nsys: 1 +hf_repo: MoE-CUDA-Graph-Analysis/tierb-b200-gbs8 +hf_remote_root: tierb_study_20260826_144644 +started: 2026-08-26T16:16:43+00:00 +driver_repo: /workspace/blackwell_run_scripts @ b99bce1 + +=== STAGE 7 -- Tier B probe and zero-drop capacity selection + +=== STEP echo__tierbprobe__gbs8 diff --git a/tierb_study_20260826_144644/tierb_timing_capacities.env.tmp.168529 b/tierb_study_20260826_144644/tierb_timing_capacities.env.tmp.168529 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..2d45a56300c533fdd7d9b7163e762370c8ddea31 100644 --- a/tierb_study_20260826_144644/tierb_timing_capacities.env.tmp.168529 +++ b/tierb_study_20260826_144644/tierb_timing_capacities.env.tmp.168529 @@ -0,0 +1,2 @@ +TIERB_TIMING_CAP_QWEN3_235B=2.00 +TIERB_TIMING_CAP_QWEN3_30B=2.00 diff --git a/tierb_study_20260826_144644/tierb_timing_capacities.tsv.tmp.168529 b/tierb_study_20260826_144644/tierb_timing_capacities.tsv.tmp.168529 index 7391632c25690f12f00cefa33868d9752c523a92..84db19b1b232a5c04d420b64dbf7737e9d8e435f 100644 --- a/tierb_study_20260826_144644/tierb_timing_capacities.tsv.tmp.168529 +++ b/tierb_study_20260826_144644/tierb_timing_capacities.tsv.tmp.168529 @@ -1 +1,3 @@ model common_timing_capacity echo_probe_required_capacity echo_full_cg_status echo_run_dir +QWEN3_235B 2.00 1.0 OK /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep3 +QWEN3_30B 2.00 1.0 OK /workspace/tierb_study_20260826_144644/echo/tierb/tierb_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep3