diff --git a/tierb-fixed-moonep-full/moonep/tierb/manifest.txt b/tierb-fixed-moonep-full/moonep/tierb/manifest.txt index 4b7096969b86911c6bf057627685547b4215c5de..fc88d6b9e48c98be2bb5eaf1d3dc4d6e61e7df95 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/manifest.txt +++ b/tierb-fixed-moonep-full/moonep/tierb/manifest.txt @@ -1,4 +1,4 @@ -timestamp: 2026-08-29T23:44:54+00:00 +timestamp: 2026-08-30T01:31:08+00:00 host: sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 gpu: NVIDIA B200 x8 driver: 580.95.05 @@ -16,4 +16,4 @@ nsys: 1 megatron_repo: /workspace/Megatron-MoE-Echo megatron_branch_required: moonep_backend_async_grad_reduce megatron_branch_actual: moonep_backend_async_grad_reduce -megatron_commit: 7bc235bf +megatron_commit: 2e68c8ad diff --git a/tierb-fixed-moonep-full/moonep/tierb/pip_freeze.txt b/tierb-fixed-moonep-full/moonep/tierb/pip_freeze.txt index cdc0e437976f3ec64164a8b07f4c534fae0a926a..75d04e488a597aae96723eef9c98b45f33b86931 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/pip_freeze.txt +++ b/tierb-fixed-moonep-full/moonep/tierb/pip_freeze.txt @@ -134,7 +134,7 @@ matplotlib==3.10.3 matplotlib-inline==0.1.7 mdit-py-plugins==0.4.2 mdurl==0.1.2 --e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@7bc235bfc2a6d28129bf8df798ac4e667113af85#egg=megatron_core +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@2e68c8ad6bfb5bac5d6eb962fa7d4fbf0a5fb095#egg=megatron_core mistune==3.1.3 mkl==2021.1.1 mkl-devel==2021.1.1 diff --git a/tierb-fixed-moonep-full/moonep/tierb/provenance.txt b/tierb-fixed-moonep-full/moonep/tierb/provenance.txt index cc96eec40523dbcfaf69a8fae14f3157aa4b01f8..110c6c9d419beef3458e6caa697571d6ab01673c 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/provenance.txt +++ b/tierb-fixed-moonep-full/moonep/tierb/provenance.txt @@ -27,3 +27,66 @@ command : run_ladder.sh --phase tierb --gbs 8 --out /workspace/tierb-fixed 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.29.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo moonep_backend_async_grad_reduce 2e68c8ad6bfb + Megatron-UltraEP ultraep-bracing-regions-through-kernels 03d57ef7d476 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-moonep eec9aee807fa + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tierb-fixed-moonep-full + MOONEP_ASSERT_NO_PENDING_REDUCE=1 + MOONEP_ASYNC_GRAD_REDUCE=1 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 29 16:41 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 29 16:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv index fca4cc3f3144522e75d1907a13899f1caf2c0087..20f6d6023f1667b9f4f108227a87256f3c64bb8d 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv @@ -7,7 +7,7 @@ 6, 0 7, 0 0, 4 -1, 1 +1, 4 2, 1 3, 1 4, 1 @@ -20,13 +20,13 @@ 3, 4 4, 4 5, 4 -6, 1 +6, 4 7, 1 0, 1 1, 1 2, 1 3, 1 -4, 3 +4, 4 5, 4 6, 4 7, 4 @@ -54,179 +54,3 @@ 5, 1 6, 1 7, 1 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 1 -6, 1 -7, 1 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 4 -7, 4 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 35 -7, 4 -0, 821 -1, 821 -2, 819 -3, 821 -4, 819 -5, 819 -6, 821 -7, 821 -0, 1537 -1, 1537 -2, 1537 -3, 1537 -4, 1537 -5, 1537 -6, 1537 -7, 1537 -0, 2471 -1, 2471 -2, 2471 -3, 2471 -4, 2471 -5, 2471 -6, 2471 -7, 2471 -0, 26195 -1, 26193 -2, 26199 -3, 26187 -4, 26235 -5, 26211 -6, 26207 -7, 26203 -0, 30319 -1, 30407 -2, 30253 -3, 30253 -4, 30253 -5, 30253 -6, 30385 -7, 30407 -0, 36177 -1, 36263 -2, 36113 -3, 36111 -4, 36111 -5, 36109 -6, 36243 -7, 36263 -0, 40669 -1, 40739 -2, 40611 -3, 40613 -4, 40591 -5, 40641 -6, 40727 -7, 40805 -0, 41133 -1, 41217 -2, 41067 -3, 41065 -4, 41065 -5, 41063 -6, 41199 -7, 41217 -0, 41137 -1, 41221 -2, 41071 -3, 41071 -4, 41071 -5, 41067 -6, 41203 -7, 41221 -0, 108757 -1, 108839 -2, 108697 -3, 108689 -4, 108699 -5, 108695 -6, 108829 -7, 108843 -0, 109391 -1, 109475 -2, 109325 -3, 109325 -4, 109325 -5, 109321 -6, 109457 -7, 109475 -0, 109665 -1, 109763 -2, 109639 -3, 109613 -4, 109633 -5, 109651 -6, 109833 -7, 109763 -0, 110429 -1, 110513 -2, 110363 -3, 110363 -4, 110363 -5, 110359 -6, 110495 -7, 110513 -0, 111079 -1, 111169 -2, 111003 -3, 110985 -4, 111047 -5, 111029 -6, 111149 -7, 111201 -0, 119291 -1, 119363 -2, 127427 -3, 121263 -4, 127429 -5, 117155 -6, 119357 -7, 119365 -0, 133237 -1, 133311 -2, 133157 -3, 133155 -4, 133155 -5, 133157 -6, 133303 -7, 133311 -0, 133237 -1, 133311 -2, 133157 -3, 133155 -4, 133155 -5, 133157 -6, 133303 -7, 133311 -0, 133237 -1, 133311 -2, 133157 -3, 133155 -4, 133155 -5, 133157 -6, 133303 -7, 133311 -0, 9575 -1, 9578 -2, 9578 -3, 9578 -4, 9898 -5, 133155 -6, 11178 -7, 9578 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env index 7992718a5f81ea3bb3c55a9b02c38dbe0e63d16e..0f958efa65262d27743e674817144c30f4eb91ff 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env @@ -3,7 +3,7 @@ config=DSV2_LITE_L24 family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log index 1b43166011ea6ad95c4ccc0896b37299fe0b52e6..6f49c116ecad12784293b1ef5f6878446f559f68 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log @@ -4,1427 +4,9 @@ L=24 seq=8192 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:52:41.339000 252492 torch/distributed/run.py:766] -W0829 23:52:41.339000 252492 torch/distributed/run.py:766] ***************************************** -W0829 23:52:41.339000 252492 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:52:41.339000 252492 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 -WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.999 - adam_eps ........................................ 1e-08 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... False - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.auto - attention_dropout ............................... 0.0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. False - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch_dist - ckpt_fully_parallel_load ........................ False - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_fusion_impl ....................... native - cross_entropy_loss_fusion ....................... False - cuda_graph_scope ................................ full_iteration - cuda_graph_warmup_steps ......................... 3 - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 8 - data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_type ................................. single - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - delay_wgrad_compute ............................. False - deprecated_use_mcore_models ..................... True - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_bf16_reduced_precision_matmul ........... False - disable_mamba_mem_eff_path ...................... False - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - embedding_init_method_std ....................... None - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... True - enable_experimental ............................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_msc ...................................... True - enable_one_logger ............................... True - encoder_num_layers .............................. 24 - encoder_seq_length .............................. 8192 - end_weight_decay ................................ 0.1 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000000 - eval_iters ...................................... 0 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 8 - expert_tensor_parallel_size ..................... 1 - export_force_local_attention .................... False - export_kd_cfg ................................... None - export_kd_teacher_ckpt_format ................... None - export_kd_teacher_load .......................... None - export_kv_cache_quant ........................... False - export_legacy_megatron .......................... False - export_model_type ............................... GPTModel - export_moe_apply_probs_on_input ................. False - export_qk_l2_norm ............................... False - export_quant_cfg ................................ None - export_real_quant_cfg ........................... None - export_te_mcore_model ........................... False - external_cuda_graph ............................. False - ffn_hidden_size ................................. 10944 - finetune ........................................ False - finetune_data_split ............................. train - finetune_hf_dataset ............................. None - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. e4m3 - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... mxfp8 - fp8_wgrad ....................................... True - fsdp_double_buffer .............................. False - global_batch_size ............................... 8 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... True - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... False - head_lr_mult .................................... 1.0 - heterogeneous_layers_config_encoded_json ........ None - heterogeneous_layers_config_path ................ None - hidden_dropout .................................. 0.0 - hidden_size ..................................... 2048 - hierarchical_context_parallel_sizes ............. None - high_priority_stream_groups ..................... [] - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_dynamic_batching ...................... False - inference_dynamic_batching_buffer_guaranteed_fraction 0.2 - inference_dynamic_batching_buffer_overflow_factor None - inference_dynamic_batching_buffer_size_gb ....... 40.0 - inference_dynamic_batching_chunk_size ........... 256 - inference_dynamic_batching_max_requests_override None - inference_dynamic_batching_max_tokens_override .. None - inference_dynamic_batching_num_cuda_graphs ...... 16 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.01 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 4294967296 - inprocess_active_world_size ..................... 8 - inprocess_barrier_timeout ....................... 120 - inprocess_completion_timeout .................... 120 - inprocess_empty_cuda_cache ...................... False - inprocess_granularity ........................... node - inprocess_hard_timeout .......................... 90 - inprocess_heartbeat_interval .................... 30 - inprocess_heartbeat_timeout ..................... 60 - inprocess_last_call_wait ........................ 1 - inprocess_max_iterations ........................ None - inprocess_monitor_process_interval .............. 1.0 - inprocess_monitor_thread_interval ............... 1.0 - inprocess_progress_watchdog_interval ............ 1.0 - inprocess_restart ............................... False - inprocess_soft_timeout .......................... 60 - inprocess_termination_grace_time ................ 1 - is_hybrid_model ................................. False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kitchen_config_file ............................. None - kitchen_recipe_number ........................... None - kv_channels ..................................... 128 - kv_lora_rank .................................... 512 - lazy_mpu_init ................................... None - load ............................................ None - load_main_params_from_ckpt ...................... None - load_model_opt_format ........................... False - local_rank ...................................... 0 - log_energy ...................................... False - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. True - log_timers_to_tensorboard ....................... False - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 8 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. None - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 5 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - mamba_head_dim .................................. 64 - mamba_num_groups ................................ 8 - mamba_num_heads ................................. None - mamba_state_dim ................................. 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... False - max_position_embeddings ......................... 32768 - max_tokens_to_oom ............................... 12000 - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - mid_level_dataset_surplus ....................... 0.005 - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-05 - mlp_chunks_for_prefill .......................... 1 - mmap_bin_files .................................. True - mock_data ....................................... False - moe_apply_probs_on_input ........................ False - moe_aux_loss_coeff .............................. 0.001 - moe_deepep_num_sms .............................. 20 - moe_echo_enable_random_offloading ............... False - moe_echo_expert_dispatch_overlap ................ False - moe_echo_expert_dispatcher_type ................. hybridep - moe_echo_recompute_expert_dispatch .............. False - moe_enable_deepep ............................... False - moe_enable_echo ................................. False - moe_enable_hybridep ............................. False - moe_enable_moonep ............................... True - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. 1408 - moe_grouped_gemm ................................ True - moe_hybridep_num_sms ............................ 16 - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] - moe_layer_recompute ............................. False - moe_moonep_async_grad_reduce .................... True - moe_moonep_num_sms .............................. None - moe_num_echo_experts ............................ None - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_received_token_capacity ..................... None - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ fp32 - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. seq_aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... True - moe_router_score_function ....................... softmax - moe_router_topk ................................. 6 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. 2816 - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... flex - moe_token_drop_policy ........................... probs - moe_upcycling_granularity ....................... 1 - moe_use_device_initiated_grouped_gemm ........... True - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mrope_section ................................... None - mscale .......................................... 0.707 - mscale_all_dim .................................. 0.707 - mtp_loss_scaling_factor ......................... 0.1 - mtp_num_layers .................................. None - multi_latent_attention .......................... True - nccl_all_reduce_for_prefill ..................... False - nccl_communicator_config_path ................... None - nccl_ub ......................................... False - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_rope_freq .................................... None - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 16 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... 64 - num_layers ...................................... 24 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_query_groups ................................ 1 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 2 - object_storage_cache_path ....................... None - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_grad_reduce ............................. True - overlap_moe_expert_parallel_comm ................ False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ True - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_layout .................. None - pipeline_model_parallel_size .................... 1 - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_l2_norm ...................................... False - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... None - recompute_method ................................ None - recompute_modules ............................... None - recompute_num_layers ............................ None - record_memory_history ........................... False - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... validate_results - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - reuse_grad_buf_for_mxfp8_param_ag ............... False - rope_scaling_factor ............................. 8.0 - rope_type ....................................... None - rotary_base ..................................... 10000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 40.0 - rotary_seq_len_interpolation_factor ............. None - run_workload_inspector_server ................... False - sample_rate ..................................... 1.0 - save ............................................ None - save_interval ................................... None - save_retain_interval ............................ None - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - seq_length ...................................... 8192 - sequence_parallel ............................... False - sft ............................................. False - sft_tokenizer_prompt_format ..................... nemotron-h-aligned - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - squared_relu .................................... False - start_weight_decay .............................. 0.1 - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - suggested_communication_unit_size ............... None - swiglu .......................................... True - swin_backbone_type .............................. tiny - symmetric_ar_type ............................... None - te_rng_tracker .................................. True - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. None - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. GPT2BPETokenizer - torch_fsdp2_reshard_after_forward ............... True - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 12 - train_samples ................................... None - train_sync_interval ............................. None - transformer_impl ................................ transformer_engine - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... True - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_sharp ....................................... False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ False - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json - vocab_size ...................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.1 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 8 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 -> building GPT2BPETokenizer tokenizer ... - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 7 -6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank -7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 2 is connected to 07[Gloo] Rank is connected to peer ranks. 17Expected number of connected peer ranks is : is connected to peer ranks. 77Expected number of connected peer ranks is : - peer ranks. 7Expected number of connected peer ranks is : -7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : -0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank7]:[W829 23:53:00.545350725 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W829 23:53:00.545361203 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:53:00.545418469 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W829 23:53:00.545488180 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W829 23:53:00.545543542 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W829 23:53:00.545610068 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:53:00.549444326 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -> compiling dataset index builder ... -make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.036 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -[rank0]:[W829 23:53:00.727658154 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.871 seconds -time to initialize megatron (seconds): 7.758 -[after megatron is initialized] datetime: 2026-08-29 23:53:06 -building GPT model ... -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1556: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( -/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. - warnings.warn( - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 -Params for bucket 1 (103022592 elements, 103022592 padded size): - module.output_layer.weight -Params for bucket 2 (48503296 elements, 48503296 padded size): - module.decoder.final_layernorm.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.23.pre_mlp_layernorm.weight - module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.23.self_attention.linear_kv_down_proj.weight - module.decoder.layers.23.self_attention.linear_q_proj.weight - module.decoder.layers.23.self_attention.linear_proj.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight -Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.mlp.router.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.21.mlp.router.weight - module.decoder.layers.21.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.22.pre_mlp_layernorm.weight - module.decoder.layers.22.self_attention.linear_kv_down_proj.weight - module.decoder.layers.22.self_attention.linear_q_proj.weight - module.decoder.layers.22.self_attention.linear_proj.weight - module.decoder.layers.22.input_layernorm.weight - module.decoder.layers.21.pre_mlp_layernorm.weight - module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.21.self_attention.linear_kv_down_proj.weight - module.decoder.layers.21.self_attention.linear_q_proj.weight -Params for bucket 4 (41163264 elements, 41163264 padded size): - module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.20.self_attention.linear_kv_down_proj.weight - module.decoder.layers.20.self_attention.linear_q_proj.weight - module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight - module.decoder.layers.21.self_attention.linear_proj.weight - module.decoder.layers.21.input_layernorm.weight - module.decoder.layers.20.mlp.router.weight - module.decoder.layers.20.self_attention.linear_kv_up_proj.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight -Params for bucket 5 (42734080 elements, 42734080 padded size): - module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.19.self_attention.linear_kv_down_proj.weight - module.decoder.layers.19.self_attention.linear_q_proj.weight - module.decoder.layers.19.self_attention.linear_proj.weight - module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.weight - module.decoder.layers.19.input_layernorm.weight -Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.18.self_attention.linear_q_proj.weight - module.decoder.layers.18.self_attention.linear_proj.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.17.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight - module.decoder.layers.18.input_layernorm.weight - module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.self_attention.linear_kv_up_proj.weight - module.decoder.layers.17.self_attention.linear_q_proj.weight -Params for bucket 7 (41163264 elements, 41163264 padded size): - module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.17.input_layernorm.weight - module.decoder.layers.16.self_attention.linear_kv_up_proj.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.16.self_attention.linear_kv_down_proj.weight - module.decoder.layers.16.self_attention.linear_q_proj.weight - module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.16.input_layernorm.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight -Params for bucket 8 (42734080 elements, 42734080 padded size): - module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.15.self_attention.linear_kv_down_proj.weight - module.decoder.layers.15.self_attention.linear_q_proj.weight - module.decoder.layers.15.self_attention.linear_proj.weight - module.decoder.layers.15.input_layernorm.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight -Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.13.self_attention.linear_kv_down_proj.weight - module.decoder.layers.13.self_attention.linear_q_proj.weight - module.decoder.layers.14.self_attention.linear_kv_down_proj.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.self_attention.linear_q_proj.weight - module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.14.input_layernorm.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.self_attention.linear_kv_up_proj.weight -Params for bucket 10 (41163264 elements, 41163264 padded size): - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.13.input_layernorm.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.self_attention.linear_kv_up_proj.weight - module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.12.self_attention.linear_kv_down_proj.weight - module.decoder.layers.12.self_attention.linear_q_proj.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.12.input_layernorm.weight -Params for bucket 11 (42734080 elements, 42734080 padded size): - module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.11.self_attention.linear_kv_down_proj.weight - module.decoder.layers.11.self_attention.linear_q_proj.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.input_layernorm.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.weight - module.decoder.layers.11.mlp.router.weight -Params for bucket 12 (40901632 elements, 40901632 padded size): - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.weight - module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.9.self_attention.linear_kv_down_proj.weight - module.decoder.layers.9.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.self_attention.linear_kv_down_proj.weight - module.decoder.layers.10.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.input_layernorm.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_kv_up_proj.weight -Params for bucket 13 (41163264 elements, 41163264 padded size): - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.9.input_layernorm.weight - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_kv_up_proj.weight - module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.pre_mlp_layernorm.weight - module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.8.self_attention.linear_kv_down_proj.weight - module.decoder.layers.8.self_attention.linear_q_proj.weight - module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.input_layernorm.weight -Params for bucket 14 (42734080 elements, 42734080 padded size): - module.decoder.layers.7.self_attention.linear_kv_up_proj.weight - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.linear_kv_down_proj.weight - module.decoder.layers.7.self_attention.linear_q_proj.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.input_layernorm.weight -Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.6.self_attention.linear_q_proj.weight - module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.6.input_layernorm.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_kv_up_proj.weight - module.decoder.layers.5.self_attention.linear_q_proj.weight -Params for bucket 16 (41163264 elements, 41163264 padded size): - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.5.input_layernorm.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.4.self_attention.linear_kv_up_proj.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.4.self_attention.linear_kv_down_proj.weight - module.decoder.layers.4.self_attention.linear_q_proj.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.4.input_layernorm.weight -Params for bucket 17 (42734080 elements, 42734080 padded size): - module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.linear_kv_down_proj.weight - module.decoder.layers.3.self_attention.linear_q_proj.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.input_layernorm.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.weight - module.decoder.layers.3.mlp.router.weight -Params for bucket 18 (40901632 elements, 40901632 padded size): - module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.2.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.linear_q_proj.weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.input_layernorm.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_kv_up_proj.weight - module.decoder.layers.1.self_attention.linear_q_proj.weight - module.decoder.layers.1.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight -Params for bucket 19 (71436288 elements, 71436288 padded size): - module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.linear_fc2.weight - module.decoder.layers.0.mlp.linear_fc1.weight - module.decoder.layers.1.input_layernorm.weight -Params for bucket 20 (116789760 elements, 116789760 padded size): - module.decoder.layers.0.self_attention.linear_q_proj.weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.weight - module.decoder.layers.0.self_attention.linear_kv_down_proj.weight - module.decoder.layers.0.input_layernorm.weight - module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.0.self_attention.linear_proj.weight - module.embedding.word_embeddings.weight -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 -Params for bucket 1 (40370176 elements, 40370176 padded size): - module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 - module.decoder.layers.23.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 - module.decoder.layers.23.mlp.experts.linear_fc2.weight5 - module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight0 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 -Params for bucket 2 (40370176 elements, 40370176 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight3 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc1.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 - module.decoder.layers.22.mlp.experts.linear_fc2.weight0 - module.decoder.layers.22.mlp.experts.linear_fc2.weight2 -Params for bucket 3 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight4 - module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight3 -Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc2.weight0 - module.decoder.layers.22.mlp.experts.linear_fc1.weight6 - module.decoder.layers.21.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc2.weight4 - module.decoder.layers.21.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 - module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.21.mlp.experts.linear_fc2.weight3 - module.decoder.layers.21.mlp.experts.linear_fc2.weight5 - module.decoder.layers.21.mlp.experts.linear_fc2.weight7 -Params for bucket 5 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc1.weight3 - module.decoder.layers.21.mlp.experts.linear_fc1.weight1 - module.decoder.layers.21.mlp.experts.linear_fc1.weight5 - module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.21.mlp.experts.linear_fc1.weight2 - module.decoder.layers.21.mlp.experts.linear_fc1.weight4 - module.decoder.layers.21.mlp.experts.linear_fc1.weight6 -Params for bucket 6 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc2.weight2 - module.decoder.layers.20.mlp.experts.linear_fc2.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc2.weight6 - module.decoder.layers.20.mlp.experts.linear_fc1.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight1 - module.decoder.layers.20.mlp.experts.linear_fc2.weight3 - module.decoder.layers.20.mlp.experts.linear_fc2.weight5 - module.decoder.layers.20.mlp.experts.linear_fc2.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight1 -Params for bucket 7 (40370176 elements, 40370176 padded size): - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.20.mlp.experts.linear_fc1.weight4 - module.decoder.layers.20.mlp.experts.linear_fc1.weight6 - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 - module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc2.weight0 -Params for bucket 8 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight3 - module.decoder.layers.19.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 -Params for bucket 9 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight4 - module.decoder.layers.18.mlp.experts.linear_fc2.weight0 - module.decoder.layers.19.mlp.experts.linear_fc1.weight6 - module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 - module.decoder.layers.18.mlp.experts.linear_fc2.weight1 - module.decoder.layers.18.mlp.experts.linear_fc2.weight3 - module.decoder.layers.18.mlp.experts.linear_fc2.weight5 -Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight6 - module.decoder.layers.18.mlp.experts.linear_fc1.weight2 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight4 - module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.18.mlp.experts.linear_fc1.weight3 - module.decoder.layers.18.mlp.experts.linear_fc1.weight5 -Params for bucket 11 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight6 - module.decoder.layers.17.mlp.experts.linear_fc2.weight2 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight1 - module.decoder.layers.18.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc2.weight3 - module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.17.mlp.experts.linear_fc2.weight7 -Params for bucket 12 (40370176 elements, 40370176 padded size): - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 - module.decoder.layers.17.mlp.experts.linear_fc1.weight4 - module.decoder.layers.17.mlp.experts.linear_fc1.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight3 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 -Params for bucket 13 (40370176 elements, 40370176 padded size): - module.decoder.layers.16.mlp.experts.linear_fc2.weight0 - module.decoder.layers.16.mlp.experts.linear_fc2.weight4 - module.decoder.layers.16.mlp.experts.linear_fc2.weight2 - module.decoder.layers.16.mlp.experts.linear_fc2.weight6 - module.decoder.layers.16.mlp.experts.linear_fc1.weight1 - module.decoder.layers.16.mlp.experts.linear_fc2.weight1 - module.decoder.layers.16.mlp.experts.linear_fc2.weight3 - module.decoder.layers.16.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc2.weight7 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 - module.decoder.layers.16.mlp.experts.linear_fc1.weight2 -Params for bucket 14 (40370176 elements, 40370176 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 -Params for bucket 15 (40370176 elements, 40370176 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight4 - module.decoder.layers.15.mlp.experts.linear_fc2.weight4 - module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight3 -Params for bucket 16 (40370176 elements, 40370176 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight6 - module.decoder.layers.14.mlp.experts.linear_fc2.weight0 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight5 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 - module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight5 - module.decoder.layers.14.mlp.experts.linear_fc2.weight7 -Params for bucket 17 (40370176 elements, 40370176 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight5 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.14.mlp.experts.linear_fc1.weight2 - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 -Params for bucket 18 (40370176 elements, 40370176 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight7 - module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 -Params for bucket 19 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 - module.decoder.layers.12.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc1.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight7 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 -Params for bucket 20 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 - module.decoder.layers.12.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 -Params for bucket 21 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 - module.decoder.layers.12.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.12.mlp.experts.linear_fc1.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 -Params for bucket 22 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 -Params for bucket 23 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 -Params for bucket 24 (40370176 elements, 40370176 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight3 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 -Params for bucket 25 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 -Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 -Params for bucket 27 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight4 -Params for bucket 28 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 -Params for bucket 29 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight5 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 -Params for bucket 30 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 -Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 -Params for bucket 32 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 -Params for bucket 33 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 -Params for bucket 34 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 -Params for bucket 35 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 -Params for bucket 36 (40370176 elements, 40370176 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 -Params for bucket 37 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 -Params for bucket 38 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 - module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 -Params for bucket 39 (40370176 elements, 40370176 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight4 -Params for bucket 40 (17301504 elements, 17301504 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:53:10 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 96 - validation: 0 - test: 0 -INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] -> building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) -INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths -INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers -INFO:megatron.core.datasets.indexed_dataset: Extract the document indices -INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 -INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 -INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices -INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy -INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy -INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 -> finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:53:10 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3648.28, 3656.20) - train/valid/test-data-iterators-setup ..........: (67.80, 78.97) -training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-29 23:53:10 -INFO:moonep.api:MoonEP comm buffer sizes: - config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 - sm_count : num_sms=32, num_sms_dedup=148 - hidden_buffer_size : 200.00 MiB - meta_buffer_size : 2.00 MiB - local_temp_buffer_size : 2.01 MiB - total_size : 204.01 MiB -[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [8832, 7168, 4224, 6272, 6528, 4736, 7296, 4480, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1], [13, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1]] -[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [3840, 4480, 5632, 5760, 7424, 5760, 5376, 6400, 5248, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[12, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1], [48, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1], [55, -1, -1, -1, -1, -1, -1, -1]] -[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [4992, 7296, 3072, 3968, 4480, 3456, 3840, 5376, 11136, 2176, 0, 0, 0, 0, 0, 0] | duplicated experts [[32, 34, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [16, -1, -1, -1, -1, -1, -1, -1]] -[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [5760, 5120, 6912, 3712, 9472, 4608, 8960, 1792, 3584, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[16, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [27, -1, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1], [7, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [29, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]] - [2026-08-29 23:53:32] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22252.1 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092134E+01 | loss scale: 1.0 | grad norm: 30.617 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 - -Number of parameters in embedding layers in billions: 0.21 -Total number of parameters in billions: 13.74 -Number of parameters in most loaded shard in billions: 13.7356 -Theoretical memory footprints: weight and optimizer=98244.96 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 34542.27197265625 | max allocated: 93366.02392578125 | reserved: 95432.0 | max reserved: 95432.0 - [2026-08-29 23:53:33] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 612.5 | throughput per GPU (TFLOP/s/GPU): 208.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.092013E+01 | loss scale: 1.0 | grad norm: 30.576 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:33] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 610.9 | throughput per GPU (TFLOP/s/GPU): 208.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952379E+00 | loss scale: 1.0 | grad norm: 42.220 | number of skipped iterations: 0 | number of nan iterations: 0 | -Capture CUDA graph for training!!! -CUDA graph capture done!!! - [2026-08-29 23:53:35] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1689.3 | throughput per GPU (TFLOP/s/GPU): 75.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477225E+00 | loss scale: 1.0 | grad norm: 7.147 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:36] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 596.8 | throughput per GPU (TFLOP/s/GPU): 213.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.121702E+01 | loss scale: 1.0 | grad norm: 398.481 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:36] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 596.5 | throughput per GPU (TFLOP/s/GPU): 213.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.369346E+00 | loss scale: 1.0 | grad norm: 5.165 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:37] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 595.7 | throughput per GPU (TFLOP/s/GPU): 214.2 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.974985E+00 | loss scale: 1.0 | grad norm: 3.703 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:37] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 595.8 | throughput per GPU (TFLOP/s/GPU): 214.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.782619E+00 | loss scale: 1.0 | grad norm: 5.638 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:38] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 596.2 | throughput per GPU (TFLOP/s/GPU): 214.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.606197E+00 | loss scale: 1.0 | grad norm: 3.129 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:39] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 595.9 | throughput per GPU (TFLOP/s/GPU): 214.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.653547E+00 | loss scale: 1.0 | grad norm: 203.613 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:39] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 596.4 | throughput per GPU (TFLOP/s/GPU): 213.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.754238E+00 | loss scale: 1.0 | grad norm: 288.991 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:53:40] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 595.7 | throughput per GPU (TFLOP/s/GPU): 214.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.575389E+00 | loss scale: 1.0 | grad norm: 70.423 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:53:40 -W0829 23:53:40.904000 252492 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:53:40.905000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252679 closing signal SIGTERM -W0829 23:53:40.905000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252680 closing signal SIGTERM -W0829 23:53:40.906000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252681 closing signal SIGTERM -W0829 23:53:40.908000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252682 closing signal SIGTERM -W0829 23:53:40.909000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252683 closing signal SIGTERM -W0829 23:53:40.910000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252684 closing signal SIGTERM -W0829 23:53:40.911000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252685 closing signal SIGTERM -W0829 23:53:40.912000 252492 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 252686 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 252492 got signal: 15 +W0830 01:38:49.752000 395171 torch/distributed/run.py:766] +W0830 01:38:49.752000 395171 torch/distributed/run.py:766] ***************************************** +W0830 01:38:49.752000 395171 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:38:49.752000 395171 torch/distributed/run.py:766] ***************************************** diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv index 090ffba8e22208c723b599f9bc8d956aea6679b1..6d30797be22863a028e082e275b06007c287303e 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv @@ -26,7 +26,7 @@ 1, 1 2, 1 3, 1 -4, 3 +4, 1 5, 4 6, 4 7, 4 @@ -62,86 +62,86 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 4 -7, 4 -0, 821 -1, 821 -2, 821 -3, 819 -4, 821 -5, 821 -6, 821 -7, 821 -0, 1537 -1, 1537 -2, 1537 -3, 1537 -4, 1537 -5, 1537 -6, 1537 -7, 1537 -0, 2551 -1, 2551 -2, 2551 -3, 2551 -4, 2551 -5, 2551 -6, 2551 -7, 2551 -0, 46603 -1, 46619 -2, 46599 -3, 46619 -4, 46635 -5, 46639 -6, 46631 -7, 46637 -0, 56813 -1, 56831 -2, 56771 -3, 56791 -4, 56519 -5, 56847 -6, 56779 -7, 56849 -0, 59169 -1, 59167 -2, 59167 -3, 59161 -4, 58875 -5, 59167 -6, 59159 -7, 59181 -0, 59733 +0, 777 +1, 751 +2, 725 +3, 767 +4, 725 +5, 731 +6, 753 +7, 725 +0, 945 +1, 943 +2, 945 +3, 939 +4, 949 +5, 963 +6, 961 +7, 957 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 32245 +1, 31861 +2, 35701 +3, 43765 +4, 35701 +5, 41461 +6, 30517 +7, 30517 +0, 48447 +1, 48449 +2, 48449 +3, 48449 +4, 48449 +5, 48449 +6, 48449 +7, 48449 +0, 56867 +1, 56821 +2, 56921 +3, 56933 +4, 56617 +5, 56883 +6, 56835 +7, 56885 +0, 58733 +1, 58755 +2, 58741 +3, 58729 +4, 58447 +5, 58749 +6, 58733 +7, 58759 +0, 59731 1, 59731 -2, 59731 -3, 59733 +2, 59733 +3, 59731 4, 59443 5, 59731 6, 59733 -7, 59731 +7, 59733 0, 59733 -1, 59731 +1, 59733 2, 59733 3, 59733 4, 59445 5, 59733 6, 59733 7, 59733 -0, 86921 -1, 86961 -2, 86961 -3, 86961 -4, 86921 -5, 86961 -6, 86961 -7, 86961 +0, 85347 +1, 85387 +2, 85387 +3, 85387 +4, 85347 +5, 85387 +6, 85387 +7, 85387 0, 87633 1, 87673 2, 87673 @@ -150,75 +150,75 @@ 5, 87673 6, 87673 7, 87673 -0, 87833 -1, 87869 -2, 87873 -3, 87873 -4, 87869 -5, 87939 -6, 87899 -7, 87937 -0, 94181 -1, 95013 -2, 95133 -3, 95229 -4, 94733 -5, 95325 -6, 94797 -7, 94821 -0, 97481 -1, 97527 -2, 97499 -3, 97465 -4, 97433 -5, 97525 -6, 97527 -7, 97523 -0, 120267 -1, 120303 -2, 120303 -3, 120303 -4, 120303 -5, 120303 -6, 120303 -7, 120303 -0, 120267 -1, 120303 -2, 120303 -3, 120303 -4, 120303 -5, 120303 -6, 120303 -7, 120303 -0, 119621 -1, 119657 -2, 119657 -3, 119657 -4, 119657 -5, 119657 -6, 119657 -7, 119657 -0, 117395 -1, 117431 -2, 117431 -3, 117431 -4, 117431 -5, 117431 -6, 117431 -7, 117431 -0, 92863 -1, 92899 -2, 92897 -3, 115751 -4, 92899 -5, 115751 -6, 115751 -7, 92899 +0, 87633 +1, 87673 +2, 87673 +3, 87673 +4, 87633 +5, 87673 +6, 87673 +7, 87673 +0, 88183 +1, 88229 +2, 88215 +3, 88229 +4, 88177 +5, 88267 +6, 88267 +7, 88259 +0, 97425 +1, 97499 +2, 97511 +3, 97499 +4, 97395 +5, 97541 +6, 97511 +7, 97625 +0, 120273 +1, 120309 +2, 120309 +3, 120309 +4, 120309 +5, 120309 +6, 120309 +7, 120309 +0, 120273 +1, 120309 +2, 120309 +3, 120309 +4, 120309 +5, 120309 +6, 120309 +7, 120309 +0, 119365 +1, 119663 +2, 119663 +3, 119663 +4, 119663 +5, 119401 +6, 119663 +7, 119663 +0, 117333 +1, 117437 +2, 117049 +3, 117501 +4, 117843 +5, 117369 +6, 117501 +7, 118083 +0, 4 +1, 115541 +2, 92905 +3, 109993 +4, 115801 +5, 92905 +6, 103465 +7, 115757 0, 1 -1, 4 +1, 3 2, 4 -3, 114021 -4, 4 +3, 4 +4, 114027 5, 4 6, 4 7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env index ae909f05330b26572108cc3340658cc4922fbe40..357f1d3998f0f6b9ed499ce2f20fda8eeb227eee 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env @@ -3,7 +3,7 @@ config=QWEN3_235B family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log index 35b4dd11a46e65a67e9955d3efd28845aa94777e..6e5056eec83f93c659da35dd7f9b93c5312219a3 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log @@ -1,23 +1,16 @@ -[23:45:03] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[01:31:18] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 L=12 seq=8192 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:45:04.937000 237461 torch/distributed/run.py:766] -W0829 23:45:04.937000 237461 torch/distributed/run.py:766] ***************************************** -W0829 23:45:04.937000 237461 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:45:04.937000 237461 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0830 01:31:19.434000 380220 torch/distributed/run.py:766] +W0830 01:31:19.434000 380220 torch/distributed/run.py:766] ***************************************** +W0830 01:31:19.434000 380220 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:31:19.434000 380220 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -575,53 +568,60 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank [Gloo] Rank 0 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 - +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 0 is connected to 72 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 5 is connected to 72 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : -7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to [Gloo] Rank 7 peer ranks. 2Expected number of connected peer ranks is : is connected to 77 - peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 00 is connected to is connected to is connected to 000 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 000 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. [Gloo] Rank peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 000 is connected to + +0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank2]:[W829 23:45:23.527949468 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W829 23:45:23.528057503 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:45:23.528113700 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W829 23:45:23.528140272 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W829 23:45:23.528167906 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:45:23.529332491 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W829 23:45:23.529419740 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 01:31:37.479166448 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 01:31:37.479214134 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 01:31:37.479251837 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 01:31:37.481705539 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 01:31:37.481736938 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 01:31:37.481915707 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 01:31:37.481926780 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds +>>> done with dataset index builder. Compilation time: 0.036 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W829 23:45:23.696190153 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.725 seconds -time to initialize megatron (seconds): 7.357 -[after megatron is initialized] datetime: 2026-08-29 23:45:29 +[rank0]:[W830 01:31:37.645502642 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.763 seconds +time to initialize megatron (seconds): 7.462 +[after megatron is initialized] datetime: 2026-08-30 01:31:43 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -646,100 +646,100 @@ Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight module.decoder.final_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight Params for bucket 3 (71835904 elements, 71835904 padded size): module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.k_layernorm.weight module.decoder.layers.10.self_attention.linear_proj.weight Params for bucket 4 (71835904 elements, 71835904 padded size): module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight Params for bucket 5 (71835904 elements, 71835904 padded size): module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight Params for bucket 6 (71835904 elements, 71835904 padded size): module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_proj.weight Params for bucket 8 (71835904 elements, 71835904 padded size): - module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_proj.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.self_attention.q_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.2.pre_mlp_layernorm.weight Params for bucket 12 (71835904 elements, 71835904 padded size): module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.k_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 13 (71835904 elements, 71835904 padded size): - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight @@ -749,8 +749,8 @@ Params for bucket 1 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 Params for bucket 2 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight10 @@ -764,12 +764,12 @@ Params for bucket 3 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight7 @@ -783,63 +783,63 @@ Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight12 Params for bucket 7 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight15 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 - module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 - module.decoder.layers.10.mlp.experts.linear_fc2.weight10 Params for bucket 9 (50331648 elements, 50331648 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight13 - module.decoder.layers.10.mlp.experts.linear_fc2.weight15 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 module.decoder.layers.10.mlp.experts.linear_fc1.weight9 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 - module.decoder.layers.10.mlp.experts.linear_fc1.weight8 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight11 Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 Params for bucket 14 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight10 - module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 @@ -870,122 +870,122 @@ Params for bucket 20 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight7 Params for bucket 21 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight8 - module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight9 module.decoder.layers.8.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight13 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 module.decoder.layers.8.mlp.experts.linear_fc1.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight9 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.8.mlp.experts.linear_fc1.weight12 Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 Params for bucket 28 (50331648 elements, 50331648 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight5 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 Params for bucket 32 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight15 Params for bucket 33 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight8 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight9 Params for bucket 34 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight10 module.decoder.layers.6.mlp.experts.linear_fc2.weight14 module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 Params for bucket 39 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 40 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 @@ -993,45 +993,45 @@ Params for bucket 40 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight14 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight15 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight11 - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 Params for bucket 45 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 Params for bucket 47 (50331648 elements, 50331648 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight12 - module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 @@ -1091,21 +1091,21 @@ Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc1.weight14 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 @@ -1128,14 +1128,14 @@ Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 Params for bucket 64 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 Params for bucket 65 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 @@ -1145,14 +1145,14 @@ Params for bucket 66 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 @@ -1160,53 +1160,53 @@ Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 - module.decoder.layers.1.mlp.experts.linear_fc1.weight13 Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 Params for bucket 71 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight1 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight8 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight9 - module.decoder.layers.0.mlp.experts.linear_fc1.weight11 - module.decoder.layers.0.mlp.experts.linear_fc1.weight10 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:45:32 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 01:31:46 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1214,7 +1214,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1227,14 +1227,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:45:32 +[after dataloaders are built] datetime: 2026-08-30 01:31:46 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3261.11, 3293.81) - train/valid/test-data-iterators-setup ..........: (63.05, 68.76) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-29 23:45:32 +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3305.74, 3311.41) + train/valid/test-data-iterators-setup ..........: (66.88, 70.10) +[before the start of training step] datetime: 2026-08-30 01:31:46 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 sm_count : num_sms=32, num_sms_dedup=148 @@ -1246,44 +1246,43 @@ INFO:moonep.api:MoonEP comm buffer sizes: [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 10496, 128, 128, 2432, 128, 1536, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [88, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [55, 52, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [124, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [58, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [111, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [121, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2304, 128, 128, 512, 128, 12416, 128, 1152, 1280, 16768, 640, 29056, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, 79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [86, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] + [2026-08-30 01:32:08] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21390.2 | throughput per GPU (TFLOP/s/GPU): 8.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB - [2026-08-29 23:45:53] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21086.4 | throughput per GPU (TFLOP/s/GPU): 8.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | [Rank 0] (after 1 iterations) memory (MB) | allocated: 67239.548828125 | max allocated: 67239.54931640625 | reserved: 68548.0 | max reserved: 68548.0 - [2026-08-29 23:45:54] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 434.3 | throughput per GPU (TFLOP/s/GPU): 416.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.335 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:54] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 331.5 | throughput per GPU (TFLOP/s/GPU): 545.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686145E+00 | loss scale: 1.0 | grad norm: 62.918 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:08] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 361.6 | throughput per GPU (TFLOP/s/GPU): 500.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.337 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:08] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 332.0 | throughput per GPU (TFLOP/s/GPU): 544.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685843E+00 | loss scale: 1.0 | grad norm: 62.891 | number of skipped iterations: 0 | number of nan iterations: 0 | Capture CUDA graph for training!!! CUDA graph capture done!!! - [2026-08-29 23:45:55] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 982.5 | throughput per GPU (TFLOP/s/GPU): 184.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338087E+01 | loss scale: 1.0 | grad norm: 20.295 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:56] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 324.9 | throughput per GPU (TFLOP/s/GPU): 556.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049608E+01 | loss scale: 1.0 | grad norm: 95.117 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:56] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 321.3 | throughput per GPU (TFLOP/s/GPU): 562.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.044566E+01 | loss scale: 1.0 | grad norm: 52.048 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:56] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 321.2 | throughput per GPU (TFLOP/s/GPU): 562.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.027068E+01 | loss scale: 1.0 | grad norm: 6.324 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:56] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 321.6 | throughput per GPU (TFLOP/s/GPU): 562.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.808611E+00 | loss scale: 1.0 | grad norm: 5.543 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:57] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 320.4 | throughput per GPU (TFLOP/s/GPU): 564.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.545443E+00 | loss scale: 1.0 | grad norm: 5.290 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:57] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 323.8 | throughput per GPU (TFLOP/s/GPU): 558.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.446625E+00 | loss scale: 1.0 | grad norm: 5.018 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:57] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 321.9 | throughput per GPU (TFLOP/s/GPU): 561.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.324770E+00 | loss scale: 1.0 | grad norm: 4.700 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:45:58] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 321.0 | throughput per GPU (TFLOP/s/GPU): 563.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.210297E+00 | loss scale: 1.0 | grad norm: 4.466 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:45:58 -[rank0]:[W829 23:45:59.838424178 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W829 23:46:00.089107186 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W829 23:46:00.110120316 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W829 23:46:00.110384142 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W829 23:46:00.117783475 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W829 23:46:00.195426955 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W829 23:46:01.738688058 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W829 23:46:01.749461216 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0829 23:46:04.688000 237461 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:46:04.689000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237651 closing signal SIGTERM -W0829 23:46:04.689000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237652 closing signal SIGTERM -W0829 23:46:04.690000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237653 closing signal SIGTERM -W0829 23:46:04.690000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237654 closing signal SIGTERM -W0829 23:46:04.690000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237655 closing signal SIGTERM -W0829 23:46:04.691000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237656 closing signal SIGTERM -W0829 23:46:04.691000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237657 closing signal SIGTERM -W0829 23:46:04.692000 237461 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 237658 closing signal SIGTERM + [2026-08-30 01:32:09] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 945.4 | throughput per GPU (TFLOP/s/GPU): 191.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338080E+01 | loss scale: 1.0 | grad norm: 20.297 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:10] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 319.2 | throughput per GPU (TFLOP/s/GPU): 566.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049032E+01 | loss scale: 1.0 | grad norm: 95.133 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:10] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 317.9 | throughput per GPU (TFLOP/s/GPU): 568.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.049771E+01 | loss scale: 1.0 | grad norm: 55.144 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:10] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 316.7 | throughput per GPU (TFLOP/s/GPU): 571.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.027093E+01 | loss scale: 1.0 | grad norm: 6.340 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:11] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 317.5 | throughput per GPU (TFLOP/s/GPU): 569.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.810117E+00 | loss scale: 1.0 | grad norm: 5.519 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:11] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 316.4 | throughput per GPU (TFLOP/s/GPU): 571.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.549583E+00 | loss scale: 1.0 | grad norm: 5.211 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:11] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 318.1 | throughput per GPU (TFLOP/s/GPU): 568.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.450918E+00 | loss scale: 1.0 | grad norm: 4.911 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:12] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 317.1 | throughput per GPU (TFLOP/s/GPU): 570.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.332552E+00 | loss scale: 1.0 | grad norm: 4.618 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:32:12] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 316.9 | throughput per GPU (TFLOP/s/GPU): 570.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.219189E+00 | loss scale: 1.0 | grad norm: 4.492 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 01:32:12 +[rank0]:[W830 01:32:13.801912097 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 01:32:14.885010634 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 01:32:14.949664138 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 01:32:14.003043426 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 01:32:14.056089290 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 01:32:15.487751536 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 01:32:15.505877593 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 01:32:15.558402976 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 01:32:19.433000 380220 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 01:32:19.433000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380405 closing signal SIGTERM +W0830 01:32:19.434000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380406 closing signal SIGTERM +W0830 01:32:19.434000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380407 closing signal SIGTERM +W0830 01:32:19.434000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380408 closing signal SIGTERM +W0830 01:32:19.434000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380409 closing signal SIGTERM +W0830 01:32:19.435000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380410 closing signal SIGTERM +W0830 01:32:19.435000 380220 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 380411 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -1311,4 +1310,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 237461 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 380220 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/mem_samples.csv index 3c2b2a41557ea13e231d09b1594c4ec79d85209e..fddcaa1db9e9289f220b6169dd892868483c1a7b 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/mem_samples.csv @@ -27,7 +27,7 @@ 2, 1 3, 1 4, 1 -5, 4 +5, 1 6, 4 7, 4 0, 0 @@ -78,52 +78,52 @@ 5, 821 6, 821 7, 821 -0, 1975 -1, 1975 -2, 1975 -3, 1975 -4, 1975 -5, 1975 -6, 1975 -7, 1975 -0, 5201 -1, 4985 -2, 4913 -3, 5249 -4, 5297 -5, 5417 -6, 4913 -7, 5153 -0, 47099 -1, 47099 -2, 47099 -3, 47099 -4, 47099 -5, 47099 -6, 47099 -7, 47099 -0, 56813 -1, 56811 -2, 56807 -3, 56801 -4, 56569 -5, 56851 -6, 56843 -7, 56875 -0, 58799 -1, 58797 -2, 58799 -3, 58797 -4, 58511 -5, 58797 -6, 58797 -7, 58799 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 1783 +4, 2551 +5, 1783 +6, 2551 +7, 2551 +0, 46595 +1, 46609 +2, 46605 +3, 46633 +4, 46595 +5, 46603 +6, 46605 +7, 46571 +0, 56769 +1, 56737 +2, 56805 +3, 56785 +4, 56561 +5, 56801 +6, 56851 +7, 56831 +0, 58731 +1, 58731 +2, 58727 +3, 58747 +4, 58469 +5, 58751 +6, 58751 +7, 58747 0, 59733 -1, 59731 -2, 59733 +1, 59733 +2, 59731 3, 59731 4, 59445 -5, 59731 +5, 59733 6, 59731 7, 59733 0, 59733 @@ -132,16 +132,24 @@ 3, 59733 4, 59445 5, 59733 -6, 59731 +6, 59733 7, 59733 -0, 86985 -1, 87025 -2, 86961 -3, 87025 -4, 86985 -5, 87025 +0, 85347 +1, 85387 +2, 85387 +3, 85387 +4, 85347 +5, 85387 6, 86961 -7, 87025 +7, 86961 +0, 87633 +1, 87673 +2, 87673 +3, 87673 +4, 87633 +5, 87673 +6, 87673 +7, 87673 0, 87633 1, 87673 2, 87673 @@ -150,74 +158,66 @@ 5, 87673 6, 87673 7, 87673 -0, 87635 -1, 87675 -2, 87675 -3, 87677 -4, 87635 -5, 87675 -6, 87675 -7, 87675 -0, 88685 -1, 88725 -2, 88725 -3, 88725 -4, 88685 -5, 88725 -6, 88725 -7, 88725 -0, 97521 -1, 97547 -2, 97525 -3, 97525 -4, 97439 -5, 97651 -6, 97549 -7, 97607 -0, 120267 -1, 120303 -2, 120303 -3, 120303 -4, 120303 -5, 120303 -6, 120303 -7, 120303 -0, 120267 -1, 120303 -2, 120303 -3, 120303 -4, 120303 -5, 120303 -6, 120303 -7, 120303 -0, 119621 -1, 119657 -2, 119657 -3, 119657 -4, 119657 -5, 119657 -6, 119657 -7, 119657 -0, 117395 -1, 117431 -2, 117819 -3, 117431 -4, 117431 -5, 117819 -6, 117431 -7, 117819 -0, 114307 -1, 114087 -2, 114669 -3, 114343 -4, 114087 -5, 114669 -6, 114407 -7, 114669 +0, 88091 +1, 88131 +2, 88129 +3, 88143 +4, 88091 +5, 88153 +6, 88149 +7, 88145 +0, 97523 +1, 97533 +2, 97555 +3, 97541 +4, 97557 +5, 97583 +6, 97567 +7, 97643 +0, 120273 +1, 120309 +2, 120309 +3, 120309 +4, 120309 +5, 120309 +6, 120309 +7, 120309 +0, 120273 +1, 120309 +2, 120309 +3, 120309 +4, 120309 +5, 120309 +6, 120309 +7, 120309 +0, 119627 +1, 119663 +2, 119663 +3, 119663 +4, 119663 +5, 119663 +6, 119663 +7, 119663 +0, 117401 +1, 117825 +2, 117437 +3, 117437 +4, 117437 +5, 117437 +6, 117437 +7, 117825 +0, 92869 +1, 115757 +2, 115757 +3, 92905 +4, 115757 +5, 115757 +6, 94633 +7, 115757 0, 1 -1, 68 -2, 114667 -3, 324 +1, 113926 +2, 4 +3, 4 4, 4 5, 4 6, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/run.env index ae909f05330b26572108cc3340658cc4922fbe40..357f1d3998f0f6b9ed499ce2f20fda8eeb227eee 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/run.env @@ -3,7 +3,7 @@ config=QWEN3_235B family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/train.log index a16ad8ec3a2c7160e9e89e926cd758a61d04eb64..cdb221b35bb8f59c0463b20d4a16c8e205dcee36 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/train.log @@ -1,25 +1,23 @@ -[23:46:17] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[01:32:32] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 L=12 seq=8192 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:46:19.086000 240032 torch/distributed/run.py:766] -W0829 23:46:19.086000 240032 torch/distributed/run.py:766] ***************************************** -W0829 23:46:19.086000 240032 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:46:19.086000 240032 torch/distributed/run.py:766] ***************************************** +W0830 01:32:33.428000 382773 torch/distributed/run.py:766] +W0830 01:32:33.428000 382773 torch/distributed/run.py:766] ***************************************** +W0830 01:32:33.428000 382773 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:32:33.428000 382773 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -577,51 +575,53 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank [Gloo] Rank 20 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 3657 is connected to is connected to is connected to is connected to 7777 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 - - - [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 4567 is connected to is connected to is connected to is connected to 7777 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7777 + + + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to 0 is connected to is connected to 0 peer ranks. 00 peer ranks. Expected number of connected peer ranks is : peer ranks. peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : +Expected number of connected peer ranks is : 000 + + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank4]:[W829 23:46:37.521085384 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W829 23:46:37.521095894 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W829 23:46:37.521112608 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:46:37.521143928 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W829 23:46:37.522800573 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W829 23:46:37.522847520 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:46:37.522872548 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 01:32:51.378489561 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 01:32:51.378526948 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 01:32:51.378555820 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 01:32:51.378565188 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 01:32:51.380630702 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 01:32:51.381181551 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 01:32:51.381335334 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W829 23:46:37.680689063 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.980 seconds -time to initialize megatron (seconds): 7.571 -[after megatron is initialized] datetime: 2026-08-29 23:46:43 +[rank0]:[W830 01:32:51.542543761 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.053 seconds +time to initialize megatron (seconds): 7.444 +[after megatron is initialized] datetime: 2026-08-30 01:32:57 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -645,43 +645,43 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.q_layernorm.weight - module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.final_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight Params for bucket 3 (71835904 elements, 71835904 padded size): - module.decoder.layers.10.mlp.router.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.q_layernorm.weight - module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.mlp.router.weight module.decoder.layers.10.self_attention.linear_proj.weight Params for bucket 4 (71835904 elements, 71835904 padded size): - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.q_layernorm.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight Params for bucket 5 (71835904 elements, 71835904 padded size): module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.q_layernorm.weight - module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight Params for bucket 6 (71835904 elements, 71835904 padded size): module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.self_attention.linear_proj.weight @@ -689,167 +689,167 @@ Params for bucket 7 (71835904 elements, 71835904 padded size): module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.q_layernorm.weight - module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight Params for bucket 8 (71835904 elements, 71835904 padded size): module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_proj.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 Params for bucket 2 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc2.weight10 - module.decoder.layers.11.mlp.experts.linear_fc2.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 Params for bucket 3 (50331648 elements, 50331648 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight14 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight8 module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight11 - module.decoder.layers.11.mlp.experts.linear_fc1.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight12 Params for bucket 7 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 Params for bucket 8 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 Params for bucket 9 (50331648 elements, 50331648 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight8 module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 Params for bucket 13 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight15 Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight8 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight10 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight12 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight8 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight9 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 @@ -857,81 +857,81 @@ Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight13 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 Params for bucket 20 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 - module.decoder.layers.8.mlp.experts.linear_fc2.weight13 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight15 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 - module.decoder.layers.8.mlp.experts.linear_fc1.weight10 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.8.mlp.experts.linear_fc1.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 27 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 Params for bucket 28 (50331648 elements, 50331648 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 Params for bucket 32 (44040192 elements, 44040192 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight14 @@ -944,37 +944,37 @@ Params for bucket 33 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight8 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight9 Params for bucket 34 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 Params for bucket 39 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 @@ -1003,78 +1003,78 @@ Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 module.decoder.layers.5.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc1.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight9 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 - module.decoder.layers.5.mlp.experts.linear_fc1.weight12 Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 46 (44040192 elements, 44040192 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 - module.decoder.layers.4.mlp.experts.linear_fc1.weight9 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight10 - module.decoder.layers.4.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 Params for bucket 51 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight10 - module.decoder.layers.3.mlp.experts.linear_fc2.weight14 - module.decoder.layers.3.mlp.experts.linear_fc2.weight12 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 @@ -1087,126 +1087,126 @@ Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight12 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight13 - module.decoder.layers.3.mlp.experts.linear_fc1.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc1.weight15 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 - module.decoder.layers.2.mlp.experts.linear_fc1.weight4 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 - module.decoder.layers.2.mlp.experts.linear_fc1.weight14 Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 Params for bucket 65 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 Params for bucket 66 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight12 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 - module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 Params for bucket 70 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 Params for bucket 71 (44040192 elements, 44040192 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight9 Params for bucket 72 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight9 - module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight10 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:46:46 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 01:33:00 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1214,7 +1214,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1227,14 +1227,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:46:47 +[after dataloaders are built] datetime: 2026-08-30 01:33:01 done with setup ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3290.03, 3304.28) - train/valid/test-data-iterators-setup ..........: (63.09, 81.05) + model-and-optimizer-setup ......................: (3407.31, 3431.39) + train/valid/test-data-iterators-setup ..........: (66.04, 78.01) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-29 23:46:47 +[before the start of training step] datetime: 2026-08-30 01:33:01 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 sm_count : num_sms=32, num_sms_dedup=148 @@ -1246,44 +1246,42 @@ INFO:moonep.api:MoonEP comm buffer sizes: [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 10496, 128, 128, 2432, 128, 1536, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [88, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [55, 52, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [124, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [58, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [111, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [121, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2304, 128, 128, 512, 128, 12416, 128, 1152, 1280, 16768, 640, 29056, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, 79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [86, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] - [2026-08-29 23:47:08] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21385.7 | throughput per GPU (TFLOP/s/GPU): 8.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.806 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:22] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 20983.8 | throughput per GPU (TFLOP/s/GPU): 8.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 67239.548828125 | max allocated: 67239.54931640625 | reserved: 68548.0 | max reserved: 68548.0 - [2026-08-29 23:47:08] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 360.7 | throughput per GPU (TFLOP/s/GPU): 501.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.336 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:09] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 332.9 | throughput per GPU (TFLOP/s/GPU): 543.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685959E+00 | loss scale: 1.0 | grad norm: 62.848 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:22] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 359.1 | throughput per GPU (TFLOP/s/GPU): 503.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.337 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:22] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 330.3 | throughput per GPU (TFLOP/s/GPU): 547.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685940E+00 | loss scale: 1.0 | grad norm: 62.876 | number of skipped iterations: 0 | number of nan iterations: 0 | Capture CUDA graph for training!!! CUDA graph capture done!!! - [2026-08-29 23:47:10] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 945.9 | throughput per GPU (TFLOP/s/GPU): 191.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338090E+01 | loss scale: 1.0 | grad norm: 20.295 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:10] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 324.3 | throughput per GPU (TFLOP/s/GPU): 557.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049949E+01 | loss scale: 1.0 | grad norm: 95.128 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:10] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 322.0 | throughput per GPU (TFLOP/s/GPU): 561.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.041523E+01 | loss scale: 1.0 | grad norm: 50.280 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:11] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 322.0 | throughput per GPU (TFLOP/s/GPU): 561.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.026906E+01 | loss scale: 1.0 | grad norm: 6.318 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:11] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 321.4 | throughput per GPU (TFLOP/s/GPU): 562.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.806656E+00 | loss scale: 1.0 | grad norm: 5.554 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:11] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 323.5 | throughput per GPU (TFLOP/s/GPU): 559.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.544450E+00 | loss scale: 1.0 | grad norm: 5.321 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:12] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 322.7 | throughput per GPU (TFLOP/s/GPU): 560.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.444943E+00 | loss scale: 1.0 | grad norm: 5.068 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:12] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 324.4 | throughput per GPU (TFLOP/s/GPU): 557.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.321287E+00 | loss scale: 1.0 | grad norm: 4.742 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:47:12] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 321.5 | throughput per GPU (TFLOP/s/GPU): 562.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.205903E+00 | loss scale: 1.0 | grad norm: 4.468 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:47:12 -[rank0]:[W829 23:47:14.239671143 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W829 23:47:15.875156992 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W829 23:47:15.875453613 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W829 23:47:15.884984177 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W829 23:47:15.886398543 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W829 23:47:15.949105908 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W829 23:47:15.956457205 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W829 23:47:15.975350193 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0829 23:47:18.883000 240032 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:47:18.884000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240216 closing signal SIGTERM -W0829 23:47:18.884000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240217 closing signal SIGTERM -W0829 23:47:18.885000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240218 closing signal SIGTERM -W0829 23:47:18.886000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240219 closing signal SIGTERM -W0829 23:47:18.887000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240220 closing signal SIGTERM -W0829 23:47:18.887000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240221 closing signal SIGTERM -W0829 23:47:18.888000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240222 closing signal SIGTERM -W0829 23:47:18.888000 240032 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 240223 closing signal SIGTERM + [2026-08-30 01:33:23] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 958.6 | throughput per GPU (TFLOP/s/GPU): 188.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338060E+01 | loss scale: 1.0 | grad norm: 20.309 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:24] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 318.8 | throughput per GPU (TFLOP/s/GPU): 567.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.050054E+01 | loss scale: 1.0 | grad norm: 95.222 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:24] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 318.0 | throughput per GPU (TFLOP/s/GPU): 568.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.042314E+01 | loss scale: 1.0 | grad norm: 50.787 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 316.6 | throughput per GPU (TFLOP/s/GPU): 571.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.026939E+01 | loss scale: 1.0 | grad norm: 6.314 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:25] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 317.1 | throughput per GPU (TFLOP/s/GPU): 570.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.807120E+00 | loss scale: 1.0 | grad norm: 5.549 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:25] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 316.5 | throughput per GPU (TFLOP/s/GPU): 571.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.543880E+00 | loss scale: 1.0 | grad norm: 5.309 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:25] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 319.1 | throughput per GPU (TFLOP/s/GPU): 566.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.444735E+00 | loss scale: 1.0 | grad norm: 5.055 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:25] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 319.8 | throughput per GPU (TFLOP/s/GPU): 565.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.322056E+00 | loss scale: 1.0 | grad norm: 4.732 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:33:26] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 317.7 | throughput per GPU (TFLOP/s/GPU): 569.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.206485E+00 | loss scale: 1.0 | grad norm: 4.466 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 01:33:26 +[rank0]:[W830 01:33:27.749790371 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 01:33:28.189206623 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 01:33:28.191561772 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 01:33:28.192020147 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 01:33:28.201417052 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 01:33:29.333803578 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 01:33:29.413464158 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 01:33:29.627573333 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 01:33:33.630000 382773 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 01:33:33.631000 382773 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 382951 closing signal SIGTERM +W0830 01:33:33.631000 382773 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 382952 closing signal SIGTERM +W0830 01:33:33.631000 382773 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 382953 closing signal SIGTERM +W0830 01:33:33.632000 382773 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 382955 closing signal SIGTERM +W0830 01:33:33.632000 382773 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 382956 closing signal SIGTERM +W0830 01:33:33.634000 382773 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 382958 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -1311,4 +1309,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 240032 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 382773 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/mem_samples.csv index 7c4869abdee3af7f4fbe001514c60cc8c9117189..59463d1c2fec8724a1a52c767bfe3197ae181345 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/mem_samples.csv @@ -50,7 +50,7 @@ 1, 4 2, 4 3, 4 -4, 1 +4, 2 5, 1 6, 1 7, 1 @@ -62,86 +62,78 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 629 -2, 47 -3, 629 -4, 45 -5, 629 -6, 629 -7, 629 -0, 909 -1, 911 -2, 931 -3, 917 -4, 937 -5, 919 -6, 951 -7, 941 -0, 1975 -1, 1975 -2, 1975 -3, 1975 -4, 1975 -5, 1975 -6, 1975 -7, 1975 -0, 36853 -1, 38389 -2, 32245 -3, 40309 -4, 46069 -5, 46453 -6, 33781 -7, 42997 -0, 48447 -1, 48449 -2, 48449 -3, 48449 -4, 48449 -5, 48449 -6, 48449 -7, 48449 -0, 56841 -1, 56871 -2, 56837 -3, 56893 -4, 56535 -5, 56877 -6, 56897 -7, 56837 -0, 59185 -1, 59185 -2, 59185 -3, 59185 -4, 58897 -5, 59185 -6, 59185 -7, 59187 +0, 819 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 2551 +1, 2551 +2, 2551 +3, 2551 +4, 2551 +5, 2551 +6, 2551 +7, 2551 +0, 46711 +1, 46701 +2, 46709 +3, 46689 +4, 46711 +5, 46711 +6, 46711 +7, 46711 +0, 56781 +1, 56813 +2, 56805 +3, 56843 +4, 56537 +5, 56875 +6, 56909 +7, 56871 +0, 58753 +1, 58717 +2, 58755 +3, 58787 +4, 58473 +5, 58753 +6, 58781 +7, 58741 0, 59731 1, 59731 -2, 59731 +2, 59733 3, 59731 4, 59443 5, 59731 6, 59731 7, 59733 0, 59733 -1, 59731 +1, 59733 2, 59733 3, 59733 4, 59445 5, 59733 -6, 59731 +6, 59733 7, 59733 -0, 86987 -1, 86961 -2, 86961 -3, 86961 -4, 86921 -5, 86961 -6, 87025 -7, 86961 +0, 85347 +1, 85387 +2, 85387 +3, 85387 +4, 85347 +5, 85387 +6, 85387 +7, 85387 0, 87633 1, 87673 2, 87673 @@ -150,75 +142,75 @@ 5, 87673 6, 87673 7, 87673 -0, 87839 -1, 87901 -2, 87905 -3, 87879 -4, 87905 -5, 87921 -6, 87937 -7, 87941 -0, 88685 -1, 88725 -2, 88725 -3, 88725 -4, 88685 -5, 88725 -6, 88725 -7, 88725 -0, 97521 -1, 97571 -2, 97513 -3, 97545 -4, 97461 -5, 97629 -6, 97583 -7, 97555 -0, 120267 -1, 120303 -2, 120303 -3, 120303 -4, 120303 -5, 120303 -6, 120303 -7, 120303 -0, 120267 -1, 120303 -2, 120303 -3, 120303 -4, 120303 -5, 120303 -6, 120303 -7, 120303 -0, 119621 -1, 119657 -2, 119657 -3, 119657 -4, 119657 -5, 119395 -6, 119657 -7, 119657 -0, 117327 -1, 117819 -2, 117495 -3, 118077 -4, 117751 -5, 117431 -6, 117495 -7, 118077 -0, 388 -1, 114667 -2, 114405 -3, 114667 -4, 114341 -5, 92897 -6, 114085 -7, 114667 +0, 87633 +1, 87673 +2, 87673 +3, 87673 +4, 87633 +5, 87673 +6, 87673 +7, 87673 +0, 88039 +1, 88079 +2, 88079 +3, 88079 +4, 88039 +5, 88079 +6, 88079 +7, 88079 +0, 97463 +1, 97489 +2, 97505 +3, 97503 +4, 97421 +5, 97551 +6, 97573 +7, 97533 +0, 120273 +1, 120309 +2, 120309 +3, 120309 +4, 120309 +5, 120309 +6, 120309 +7, 120309 +0, 120273 +1, 120309 +2, 120309 +3, 120309 +4, 120309 +5, 120309 +6, 120309 +7, 120309 +0, 119045 +1, 119663 +2, 119663 +3, 119663 +4, 119663 +5, 119663 +6, 119337 +7, 119017 +0, 117077 +1, 118083 +2, 117433 +3, 118083 +4, 117757 +5, 117437 +6, 117437 +7, 116787 +0, 92867 +1, 115757 +2, 94633 +3, 115757 +4, 115757 +5, 115757 +6, 110761 +7, 4 0, 1 -1, 1 -2, 3 -3, 4 -4, 114021 +1, 4 +2, 4 +3, 114027 +4, 4 5, 4 6, 4 7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/run.env index ae909f05330b26572108cc3340658cc4922fbe40..357f1d3998f0f6b9ed499ce2f20fda8eeb227eee 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/run.env @@ -3,7 +3,7 @@ config=QWEN3_235B family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/train.log index d35d58b6abfa58614b952e8de9ede45267ed3eec..ea03df418621ae7adc738981f887ff0f019ada37 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/train.log @@ -1,25 +1,23 @@ -[23:47:32] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[01:33:45] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) ============================================================== QWEN3_235B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 L=12 seq=8192 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:47:34.311000 242741 torch/distributed/run.py:766] -W0829 23:47:34.311000 242741 torch/distributed/run.py:766] ***************************************** -W0829 23:47:34.311000 242741 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:47:34.311000 242741 torch/distributed/run.py:766] ***************************************** +W0830 01:33:47.017000 385325 torch/distributed/run.py:766] +W0830 01:33:47.017000 385325 torch/distributed/run.py:766] ***************************************** +W0830 01:33:47.017000 385325 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:33:47.017000 385325 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -574,54 +572,56 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +[Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 0 is connected to 17 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : [Gloo] Rank -72 is connected to -7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 54 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to 7 peer ranks. peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to 0 is connected to 0 is connected to 0 peer ranks. 0 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 0Expected number of connected peer ranks is : 0 -0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 +[Gloo] Rank 0[Gloo] Rank is connected to 00[Gloo] Rank is connected to peer ranks. 00Expected number of connected peer ranks is : is connected to peer ranks. 00Expected number of connected peer ranks is : + peer ranks. 0Expected number of connected peer ranks is : +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank2]:[W829 23:47:52.047774281 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W829 23:47:52.064582906 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W829 23:47:52.065910975 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W829 23:47:52.065942810 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:47:52.066506087 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:47:52.066710093 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W829 23:47:52.068582353 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 01:34:04.105198961 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 01:34:04.105212291 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 01:34:04.105213047 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 01:34:04.105279090 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 01:34:04.105289555 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 01:34:04.105317293 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 01:34:04.105351538 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W829 23:47:53.247994951 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.776 seconds -time to initialize megatron (seconds): 7.432 -[after megatron is initialized] datetime: 2026-08-29 23:47:58 +[rank0]:[W830 01:34:05.270035787 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.693 seconds +time to initialize megatron (seconds): 7.525 +[after megatron is initialized] datetime: 2026-08-30 01:34:10 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -645,36 +645,36 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): - module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.final_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_proj.weight Params for bucket 4 (71835904 elements, 71835904 padded size): module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight Params for bucket 5 (71835904 elements, 71835904 padded size): module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.self_attention.q_layernorm.weight - module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight Params for bucket 6 (71835904 elements, 71835904 padded size): @@ -683,62 +683,62 @@ Params for bucket 6 (71835904 elements, 71835904 padded size): module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.7.self_attention.q_layernorm.weight - module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.self_attention.q_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.6.self_attention.linear_proj.weight Params for bucket 8 (71835904 elements, 71835904 padded size): - module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.linear_qkv.weight module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_proj.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.k_layernorm.weight - module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 13 (71835904 elements, 71835904 padded size): - module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.mlp.router.weight module.decoder.layers.0.self_attention.q_layernorm.weight - module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight Params for bucket 14 (206045184 elements, 206045184 padded size): @@ -746,8 +746,8 @@ Params for bucket 14 (206045184 elements, 206045184 padded size): INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 @@ -757,40 +757,40 @@ Params for bucket 2 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight13 Params for bucket 3 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight0 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight10 Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight12 module.decoder.layers.11.mlp.experts.linear_fc1.weight13 - module.decoder.layers.11.mlp.experts.linear_fc1.weight14 Params for bucket 7 (44040192 elements, 44040192 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 Params for bucket 8 (44040192 elements, 44040192 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 @@ -798,68 +798,68 @@ Params for bucket 8 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight11 Params for bucket 9 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight8 module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 Params for bucket 13 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 Params for bucket 14 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight10 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight12 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight12 module.decoder.layers.9.mlp.experts.linear_fc1.weight10 - module.decoder.layers.9.mlp.experts.linear_fc1.weight11 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight13 - module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 Params for bucket 20 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 @@ -873,286 +873,286 @@ Params for bucket 21 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight9 module.decoder.layers.8.mlp.experts.linear_fc2.weight13 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 - module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight8 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 Params for bucket 28 (50331648 elements, 50331648 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 module.decoder.layers.7.mlp.experts.linear_fc1.weight13 Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight15 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 Params for bucket 33 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 Params for bucket 34 (50331648 elements, 50331648 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight13 module.decoder.layers.6.mlp.experts.linear_fc1.weight15 - module.decoder.layers.6.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 Params for bucket 39 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 - module.decoder.layers.5.mlp.experts.linear_fc2.weight13 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight9 module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight11 - module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight12 module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 Params for bucket 45 (44040192 elements, 44040192 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.4.mlp.experts.linear_fc2.weight10 Params for bucket 47 (50331648 elements, 50331648 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc2.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight6 - module.decoder.layers.4.mlp.experts.linear_fc1.weight9 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight10 - module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight13 module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 Params for bucket 52 (44040192 elements, 44040192 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 Params for bucket 53 (50331648 elements, 50331648 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight10 - module.decoder.layers.3.mlp.experts.linear_fc2.weight12 - module.decoder.layers.3.mlp.experts.linear_fc2.weight14 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 - module.decoder.layers.3.mlp.experts.linear_fc1.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight15 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight3 - module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight10 - module.decoder.layers.2.mlp.experts.linear_fc1.weight8 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight14 - module.decoder.layers.2.mlp.experts.linear_fc1.weight12 Params for bucket 64 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 Params for bucket 65 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight14 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 @@ -1160,53 +1160,53 @@ Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight10 - module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 Params for bucket 70 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 Params for bucket 71 (44040192 elements, 44040192 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 - module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 Params for bucket 72 (50331648 elements, 50331648 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight1 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight5 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc1.weight8 module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight9 - module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:48:02 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 01:34:14 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1214,7 +1214,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1227,14 +1227,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:48:02 +[after dataloaders are built] datetime: 2026-08-30 01:34:14 done with setup ... training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3473.33, 3497.43) - train/valid/test-data-iterators-setup ..........: (64.41, 69.66) -[before the start of training step] datetime: 2026-08-29 23:48:02 + model-and-optimizer-setup ......................: (3338.00, 3360.91) + train/valid/test-data-iterators-setup ..........: (63.69, 75.16) +[before the start of training step] datetime: 2026-08-30 01:34:14 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 sm_count : num_sms=32, num_sms_dedup=148 @@ -1246,44 +1246,42 @@ INFO:moonep.api:MoonEP comm buffer sizes: [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 10496, 128, 128, 2432, 128, 1536, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [88, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [55, 52, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [124, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [58, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [111, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [121, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2304, 128, 128, 512, 128, 12416, 128, 1152, 1280, 16768, 640, 29056, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, 79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [86, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] - [2026-08-29 23:48:23] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21162.6 | throughput per GPU (TFLOP/s/GPU): 8.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.807 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:35] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21275.7 | throughput per GPU (TFLOP/s/GPU): 8.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.806 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 Theoretical memory footprints: weight and optimizer=216428.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 67239.548828125 | max allocated: 67239.54931640625 | reserved: 68548.0 | max reserved: 68548.0 - [2026-08-29 23:48:24] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 436.6 | throughput per GPU (TFLOP/s/GPU): 414.1 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.335 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:24] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 331.5 | throughput per GPU (TFLOP/s/GPU): 545.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685947E+00 | loss scale: 1.0 | grad norm: 62.878 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:36] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 364.0 | throughput per GPU (TFLOP/s/GPU): 496.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.336 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:36] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 331.4 | throughput per GPU (TFLOP/s/GPU): 545.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.686085E+00 | loss scale: 1.0 | grad norm: 62.885 | number of skipped iterations: 0 | number of nan iterations: 0 | Capture CUDA graph for training!!! CUDA graph capture done!!! - [2026-08-29 23:48:25] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 944.5 | throughput per GPU (TFLOP/s/GPU): 191.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338058E+01 | loss scale: 1.0 | grad norm: 20.306 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:25] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 324.9 | throughput per GPU (TFLOP/s/GPU): 556.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049307E+01 | loss scale: 1.0 | grad norm: 95.187 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:26] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 322.5 | throughput per GPU (TFLOP/s/GPU): 560.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.048625E+01 | loss scale: 1.0 | grad norm: 54.475 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:26] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 323.2 | throughput per GPU (TFLOP/s/GPU): 559.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.027090E+01 | loss scale: 1.0 | grad norm: 6.334 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:26] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 322.7 | throughput per GPU (TFLOP/s/GPU): 560.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.809750E+00 | loss scale: 1.0 | grad norm: 5.526 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:27] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 321.8 | throughput per GPU (TFLOP/s/GPU): 562.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.547915E+00 | loss scale: 1.0 | grad norm: 5.230 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:27] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 322.3 | throughput per GPU (TFLOP/s/GPU): 561.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.449049E+00 | loss scale: 1.0 | grad norm: 4.934 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:27] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 320.9 | throughput per GPU (TFLOP/s/GPU): 563.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.330864E+00 | loss scale: 1.0 | grad norm: 4.634 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:48:28] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 321.9 | throughput per GPU (TFLOP/s/GPU): 561.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.217180E+00 | loss scale: 1.0 | grad norm: 4.483 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:48:28 -[rank0]:[W829 23:48:29.704910691 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W829 23:48:30.362260925 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W829 23:48:30.756421216 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W829 23:48:31.400079860 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W829 23:48:31.439735903 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W829 23:48:31.441838729 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W829 23:48:31.444486514 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W829 23:48:31.448760172 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0829 23:48:34.082000 242741 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:48:34.082000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242931 closing signal SIGTERM -W0829 23:48:34.083000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242932 closing signal SIGTERM -W0829 23:48:34.083000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242933 closing signal SIGTERM -W0829 23:48:34.084000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242934 closing signal SIGTERM -W0829 23:48:34.084000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242935 closing signal SIGTERM -W0829 23:48:34.085000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242936 closing signal SIGTERM -W0829 23:48:34.085000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242937 closing signal SIGTERM -W0829 23:48:34.086000 242741 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 242938 closing signal SIGTERM + [2026-08-30 01:34:37] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 979.0 | throughput per GPU (TFLOP/s/GPU): 184.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338064E+01 | loss scale: 1.0 | grad norm: 20.299 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:37] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 319.7 | throughput per GPU (TFLOP/s/GPU): 565.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049333E+01 | loss scale: 1.0 | grad norm: 95.200 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:38] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 318.9 | throughput per GPU (TFLOP/s/GPU): 567.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.048383E+01 | loss scale: 1.0 | grad norm: 54.327 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:38] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 318.4 | throughput per GPU (TFLOP/s/GPU): 568.0 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.027052E+01 | loss scale: 1.0 | grad norm: 6.338 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:38] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 318.4 | throughput per GPU (TFLOP/s/GPU): 567.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.809936E+00 | loss scale: 1.0 | grad norm: 5.526 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:38] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 317.7 | throughput per GPU (TFLOP/s/GPU): 569.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.548426E+00 | loss scale: 1.0 | grad norm: 5.232 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:39] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 318.2 | throughput per GPU (TFLOP/s/GPU): 568.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.449459E+00 | loss scale: 1.0 | grad norm: 4.938 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:39] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 318.6 | throughput per GPU (TFLOP/s/GPU): 567.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.331503E+00 | loss scale: 1.0 | grad norm: 4.636 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:34:39] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 318.2 | throughput per GPU (TFLOP/s/GPU): 568.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.217281E+00 | loss scale: 1.0 | grad norm: 4.482 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 01:34:39 +[rank0]:[W830 01:34:41.434365646 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 01:34:42.271159288 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 01:34:42.512519329 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 01:34:42.516537505 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 01:34:42.636432904 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 01:34:42.082170425 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 01:34:42.086366756 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 01:34:42.088352473 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 01:34:46.823000 385325 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 01:34:46.823000 385325 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 385510 closing signal SIGTERM +W0830 01:34:46.824000 385325 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 385511 closing signal SIGTERM +W0830 01:34:46.824000 385325 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 385512 closing signal SIGTERM +W0830 01:34:46.825000 385325 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 385513 closing signal SIGTERM +W0830 01:34:46.825000 385325 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 385514 closing signal SIGTERM +W0830 01:34:46.826000 385325 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 385515 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -1311,4 +1309,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 242741 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 385325 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv index 142e2a881e4983d79326909f97b6402c1fed86c3..ea9523e430d7afa2970383489dde2c4409784718 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv @@ -20,13 +20,13 @@ 3, 4 4, 4 5, 4 -6, 2 +6, 4 7, 1 0, 1 1, 1 2, 1 3, 1 -4, 3 +4, 1 5, 4 6, 4 7, 4 @@ -45,20 +45,12 @@ 4, 1 5, 1 6, 1 -7, 0 -0, 4 -1, 1 -2, 1 -3, 1 -4, 1 -5, 1 -6, 1 7, 1 0, 4 1, 4 2, 4 3, 4 -4, 4 +4, 1 5, 1 6, 1 7, 1 @@ -70,22 +62,22 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 47 -2, 47 -3, 47 -4, 629 -5, 629 -6, 629 -7, 629 -0, 911 -1, 919 -2, 905 -3, 929 -4, 909 -5, 929 -6, 925 -7, 919 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1079 +1, 1077 +2, 1079 +3, 1079 +4, 1079 +5, 1079 +6, 1079 +7, 1079 0, 1831 1, 1831 2, 1831 @@ -94,14 +86,14 @@ 5, 1831 6, 1831 7, 1831 -0, 12661 -1, 12677 -2, 12461 -3, 13213 -4, 13017 -5, 12697 -6, 12877 -7, 13133 +0, 19271 +1, 19269 +2, 19277 +3, 19283 +4, 19291 +5, 19287 +6, 19291 +7, 19277 0, 47207 1, 47207 2, 47207 @@ -110,7 +102,7 @@ 5, 47207 6, 47207 7, 47207 -0, 47547 +0, 47549 1, 47551 2, 47551 3, 47551 @@ -118,54 +110,54 @@ 5, 47551 6, 47551 7, 47551 -0, 48133 -1, 47959 -2, 48161 -3, 47959 -4, 47959 -5, 48161 -6, 47959 -7, 48161 -0, 48721 -1, 48375 -2, 48749 -3, 48375 -4, 48511 -5, 48749 -6, 48375 -7, 48749 -0, 49251 +0, 48313 +1, 47969 +2, 48341 +3, 47969 +4, 48105 +5, 48341 +6, 47969 +7, 48341 +0, 49103 +1, 48759 +2, 49131 +3, 48759 +4, 48895 +5, 49131 +6, 48761 +7, 49131 +0, 49249 1, 48905 -2, 49279 +2, 49277 3, 48905 4, 49041 -5, 49279 -6, 48905 -7, 49279 +5, 49277 +6, 48907 +7, 49277 0, 49251 -1, 48905 +1, 48907 2, 49279 -3, 48907 +3, 48905 4, 49043 -5, 49279 +5, 49277 6, 48907 7, 49279 -0, 62095 -1, 62075 -2, 62089 -3, 62079 -4, 62107 -5, 62089 -6, 62079 -7, 62089 -0, 83419 -1, 83473 -2, 83455 -3, 83423 -4, 83495 -5, 83463 -6, 83419 -7, 83453 +0, 62969 +1, 62867 +2, 62963 +3, 62953 +4, 62981 +5, 62963 +6, 62953 +7, 62963 +0, 83461 +1, 83465 +2, 83463 +3, 83461 +4, 83517 +5, 83475 +6, 83467 +7, 83469 0, 83947 1, 83951 2, 83941 @@ -174,67 +166,67 @@ 5, 83941 6, 83931 7, 83941 -0, 84229 -1, 84255 -2, 84241 -3, 84233 -4, 84349 -5, 84291 -6, 84251 -7, 84331 -0, 84613 -1, 84617 -2, 84607 -3, 84597 -4, 84625 -5, 84607 -6, 84597 -7, 84607 -0, 95193 -1, 95129 -2, 95197 -3, 94989 -4, 95123 -5, 95213 -6, 95089 -7, 95221 -0, 116439 -1, 116343 -2, 116411 -3, 116193 -4, 116315 -5, 116411 -6, 116289 -7, 116411 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116551 -7, 116701 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116555 -7, 116701 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116555 -7, 116701 -0, 21171 -1, 21174 -2, 21494 -3, 116457 -4, 22774 +0, 84269 +1, 84223 +2, 84199 +3, 84223 +4, 84339 +5, 84293 +6, 84327 +7, 84265 +0, 89737 +1, 89989 +2, 89763 +3, 89657 +4, 89661 +5, 89907 +6, 89813 +7, 90003 +0, 95559 +1, 95447 +2, 95535 +3, 95295 +4, 95449 +5, 95557 +6, 95403 +7, 95569 +0, 105447 +1, 106009 +2, 105845 +3, 107203 +4, 106127 +5, 106645 +6, 105807 +7, 111847 +0, 116707 +1, 116615 +2, 116705 +3, 116463 +4, 116587 +5, 116705 +6, 116559 +7, 116705 +0, 116707 +1, 116615 +2, 116705 +3, 116463 +4, 116587 +5, 116705 +6, 116559 +7, 116705 +0, 115739 +1, 115967 +2, 116057 +3, 115815 +4, 115939 +5, 116057 +6, 115911 +7, 115795 +0, 21173 +1, 21430 +2, 116057 +3, 22454 +4, 21174 5, 21174 6, 21174 7, 21174 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env index 4d121388cdc2db102c1783c999c4de53c0e99bb8..2e2a2fd96c750ea9322cc0a5bc54f2c95e45a4b6 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env @@ -3,7 +3,7 @@ config=QWEN3_30B family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log index de25988a22eebf966b8670a5e25f3b60f2467a56..cbb58d4827e0f887c2aea7a6c2d0a4077eae95a0 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log @@ -1,16 +1,23 @@ -[23:48:48] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +[01:34:59] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) ============================================================== QWEN3_30B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=768 m=16 L=48 seq=4096 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:48:49.788000 245313 torch/distributed/run.py:766] -W0829 23:48:49.788000 245313 torch/distributed/run.py:766] ***************************************** -W0829 23:48:49.788000 245313 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:48:49.788000 245313 torch/distributed/run.py:766] ***************************************** +W0830 01:35:01.054000 387880 torch/distributed/run.py:766] +W0830 01:35:01.054000 387880 torch/distributed/run.py:766] ***************************************** +W0830 01:35:01.054000 387880 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:35:01.054000 387880 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -563,7 +570,6 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -571,46 +577,40 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 27 is connected to +7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 7 +3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 35 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - -[Gloo] Rank [Gloo] Rank 0 is connected to 47 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0[Gloo] Rank is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank4]:[W829 23:49:08.832191331 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W829 23:49:08.832191210 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W829 23:49:08.832254365 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:49:08.832338585 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W829 23:49:08.832611246 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:49:08.834256262 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W829 23:49:08.836157723 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 01:35:18.070365871 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 01:35:18.070922035 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 01:35:18.080020803 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 01:35:18.080043323 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 01:35:18.080047040 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 01:35:18.080139098 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 01:35:18.080526533 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -618,10 +618,10 @@ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W829 23:49:08.998795269 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.615 seconds -time to initialize megatron (seconds): 7.128 -[after megatron is initialized] datetime: 2026-08-29 23:49:14 +[rank0]:[W830 01:35:19.245892754 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.833 seconds +time to initialize megatron (seconds): 7.622 +[after megatron is initialized] datetime: 2026-08-30 01:35:25 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -645,184 +645,176 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (49033984 elements, 49033984 padded size): - module.decoder.layers.47.self_attention.k_layernorm.weight module.decoder.layers.47.pre_mlp_layernorm.weight module.decoder.layers.46.pre_mlp_layernorm.weight - module.decoder.layers.46.self_attention.k_layernorm.weight - module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.45.mlp.router.weight - module.decoder.layers.47.self_attention.q_layernorm.weight + module.decoder.layers.47.self_attention.k_layernorm.weight module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.q_layernorm.weight module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.47.self_attention.linear_proj.weight module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.46.self_attention.linear_proj.weight module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.47.self_attention.linear_proj.weight + module.decoder.layers.46.self_attention.linear_proj.weight module.decoder.layers.45.self_attention.k_layernorm.weight module.decoder.layers.45.self_attention.linear_qkv.weight - module.decoder.layers.47.mlp.router.weight - module.decoder.final_layernorm.weight module.decoder.layers.46.mlp.router.weight module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.47.mlp.router.weight Params for bucket 3 (46672384 elements, 46672384 padded size): - module.decoder.layers.44.self_attention.k_layernorm.weight - module.decoder.layers.44.pre_mlp_layernorm.weight - module.decoder.layers.44.mlp.router.weight - module.decoder.layers.44.self_attention.q_layernorm.weight - module.decoder.layers.44.self_attention.linear_qkv.weight - module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.44.self_attention.linear_proj.weight module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.45.self_attention.linear_proj.weight module.decoder.layers.43.mlp.router.weight module.decoder.layers.43.pre_mlp_layernorm.weight module.decoder.layers.43.self_attention.k_layernorm.weight - module.decoder.layers.43.self_attention.q_layernorm.weight module.decoder.layers.43.self_attention.linear_qkv.weight module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.44.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.self_attention.q_layernorm.weight + module.decoder.layers.44.self_attention.linear_qkv.weight + module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.self_attention.q_layernorm.weight Params for bucket 4 (49031936 elements, 49031936 padded size): module.decoder.layers.42.pre_mlp_layernorm.weight module.decoder.layers.41.pre_mlp_layernorm.weight - module.decoder.layers.41.self_attention.k_layernorm.weight - module.decoder.layers.41.self_attention.linear_qkv.weight module.decoder.layers.40.mlp.router.weight - module.decoder.layers.42.self_attention.linear_qkv.weight - module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.self_attention.k_layernorm.weight module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.pre_mlp_layernorm.weight module.decoder.layers.42.self_attention.linear_proj.weight module.decoder.layers.41.self_attention.linear_proj.weight - module.decoder.layers.40.pre_mlp_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight module.decoder.layers.42.mlp.router.weight - module.decoder.layers.40.self_attention.q_layernorm.weight module.decoder.layers.41.mlp.router.weight - module.decoder.layers.40.self_attention.k_layernorm.weight module.decoder.layers.40.self_attention.linear_qkv.weight Params for bucket 5 (46672384 elements, 46672384 padded size): + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_proj.weight module.decoder.layers.39.mlp.router.weight + module.decoder.layers.39.self_attention.k_layernorm.weight module.decoder.layers.40.self_attention.linear_proj.weight module.decoder.layers.39.pre_mlp_layernorm.weight - module.decoder.layers.39.self_attention.k_layernorm.weight module.decoder.layers.39.self_attention.q_layernorm.weight module.decoder.layers.39.self_attention.linear_qkv.weight module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.39.self_attention.linear_proj.weight - module.decoder.layers.38.mlp.router.weight - module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.38.pre_mlp_layernorm.weight - module.decoder.layers.38.self_attention.k_layernorm.weight module.decoder.layers.38.self_attention.q_layernorm.weight - module.decoder.layers.38.self_attention.linear_qkv.weight module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.38.self_attention.linear_proj.weight Params for bucket 6 (49031936 elements, 49031936 padded size): module.decoder.layers.37.pre_mlp_layernorm.weight module.decoder.layers.36.pre_mlp_layernorm.weight - module.decoder.layers.35.mlp.router.weight module.decoder.layers.35.self_attention.k_layernorm.weight module.decoder.layers.35.self_attention.linear_qkv.weight - module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.k_layernorm.weight - module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.35.mlp.router.weight module.decoder.layers.35.self_attention.q_layernorm.weight module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.36.self_attention.k_layernorm.weight module.decoder.layers.36.self_attention.q_layernorm.weight module.decoder.layers.36.self_attention.linear_qkv.weight module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.37.self_attention.linear_proj.weight module.decoder.layers.36.self_attention.linear_proj.weight module.decoder.layers.35.pre_mlp_layernorm.weight - module.decoder.layers.36.mlp.router.weight module.decoder.layers.37.mlp.router.weight + module.decoder.layers.36.mlp.router.weight Params for bucket 7 (46672384 elements, 46672384 padded size): - module.decoder.layers.35.self_attention.linear_proj.weight - module.decoder.layers.33.mlp.router.weight - module.decoder.layers.33.pre_mlp_layernorm.weight - module.decoder.layers.33.self_attention.k_layernorm.weight - module.decoder.layers.33.self_attention.q_layernorm.weight - module.decoder.layers.33.self_attention.linear_qkv.weight - module.decoder.layers.33.self_attention.linear_proj.weight module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.34.pre_mlp_layernorm.weight - module.decoder.layers.34.mlp.router.weight module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.35.self_attention.linear_proj.weight module.decoder.layers.34.self_attention.q_layernorm.weight module.decoder.layers.34.self_attention.linear_qkv.weight module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.pre_mlp_layernorm.weight + module.decoder.layers.33.self_attention.k_layernorm.weight + module.decoder.layers.33.self_attention.linear_qkv.weight module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.33.self_attention.q_layernorm.weight Params for bucket 8 (49031936 elements, 49031936 padded size): - module.decoder.layers.32.mlp.router.weight module.decoder.layers.32.pre_mlp_layernorm.weight module.decoder.layers.31.pre_mlp_layernorm.weight module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.k_layernorm.weight module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight module.decoder.layers.30.self_attention.k_layernorm.weight module.decoder.layers.30.self_attention.linear_qkv.weight - module.decoder.layers.32.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_qkv.weight - module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.31.self_attention.q_layernorm.weight - module.decoder.layers.31.self_attention.linear_qkv.weight - module.decoder.layers.30.mlp.router.weight module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.mlp.router.weight module.decoder.layers.30.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_proj.weight module.decoder.layers.31.self_attention.linear_proj.weight module.decoder.layers.31.mlp.router.weight + module.decoder.layers.32.mlp.router.weight Params for bucket 9 (46672384 elements, 46672384 padded size): module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.pre_mlp_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight module.decoder.layers.28.self_attention.q_layernorm.weight module.decoder.layers.28.self_attention.linear_qkv.weight module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.29.mlp.router.weight - module.decoder.layers.29.self_attention.k_layernorm.weight - module.decoder.layers.29.pre_mlp_layernorm.weight module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.29.pre_mlp_layernorm.weight + module.decoder.layers.29.self_attention.k_layernorm.weight module.decoder.layers.29.self_attention.q_layernorm.weight module.decoder.layers.29.self_attention.linear_qkv.weight module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.29.self_attention.linear_proj.weight - module.decoder.layers.28.pre_mlp_layernorm.weight - module.decoder.layers.28.self_attention.k_layernorm.weight module.decoder.layers.28.self_attention.linear_proj.weight Params for bucket 10 (49031936 elements, 49031936 padded size): module.decoder.layers.27.mlp.router.weight + module.decoder.layers.26.mlp.router.weight module.decoder.layers.26.pre_mlp_layernorm.weight module.decoder.layers.25.pre_mlp_layernorm.weight module.decoder.layers.27.pre_mlp_layernorm.weight module.decoder.layers.26.self_attention.k_layernorm.weight - module.decoder.layers.26.self_attention.linear_qkv.weight module.decoder.layers.25.self_attention.k_layernorm.weight module.decoder.layers.27.self_attention.linear_qkv.weight module.decoder.layers.27.self_attention.k_layernorm.weight module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.27.self_attention.q_layernorm.weight module.decoder.layers.27.self_attention.linear_proj.weight module.decoder.layers.26.self_attention.linear_proj.weight module.decoder.layers.25.mlp.router.weight - module.decoder.layers.26.mlp.router.weight Params for bucket 11 (46672384 elements, 46672384 padded size): module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.24.pre_mlp_layernorm.weight - module.decoder.layers.24.self_attention.k_layernorm.weight - module.decoder.layers.24.mlp.router.weight - module.decoder.layers.24.self_attention.q_layernorm.weight - module.decoder.layers.24.self_attention.linear_qkv.weight - module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.24.self_attention.linear_proj.weight - module.decoder.layers.25.self_attention.linear_proj.weight module.decoder.layers.23.mlp.router.weight module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.self_attention.k_layernorm.weight @@ -830,17 +822,25 @@ Params for bucket 11 (46672384 elements, 46672384 padded size): module.decoder.layers.23.self_attention.linear_qkv.weight module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.k_layernorm.weight + module.decoder.layers.24.mlp.router.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.self_attention.q_layernorm.weight + module.decoder.layers.24.self_attention.linear_qkv.weight + module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.24.self_attention.linear_proj.weight Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.22.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.20.self_attention.k_layernorm.weight module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.21.self_attention.k_layernorm.weight - module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.k_layernorm.weight module.decoder.layers.22.self_attention.k_layernorm.weight module.decoder.layers.22.self_attention.linear_qkv.weight module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.20.self_attention.q_layernorm.weight module.decoder.layers.20.self_attention.linear_qkv.weight @@ -852,14 +852,8 @@ Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.20.mlp.router.weight Params for bucket 13 (46672384 elements, 46672384 padded size): module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.19.self_attention.k_layernorm.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.self_attention.linear_qkv.weight - module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.19.self_attention.q_layernorm.weight + module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.18.mlp.router.weight module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.self_attention.k_layernorm.weight @@ -867,26 +861,32 @@ Params for bucket 13 (46672384 elements, 46672384 padded size): module.decoder.layers.18.self_attention.linear_qkv.weight module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight Params for bucket 14 (49031936 elements, 49031936 padded size): module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight module.decoder.layers.16.self_attention.k_layernorm.weight - module.decoder.layers.16.self_attention.linear_qkv.weight - module.decoder.layers.15.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.weight module.decoder.layers.17.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.k_layernorm.weight module.decoder.layers.15.self_attention.q_layernorm.weight module.decoder.layers.15.self_attention.linear_qkv.weight module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.linear_proj.weight module.decoder.layers.17.self_attention.linear_proj.weight module.decoder.layers.16.mlp.router.weight - module.decoder.layers.15.mlp.router.weight Params for bucket 15 (46672384 elements, 46672384 padded size): module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.14.self_attention.q_layernorm.weight @@ -898,79 +898,72 @@ Params for bucket 15 (46672384 elements, 46672384 padded size): module.decoder.layers.13.self_attention.q_layernorm.weight module.decoder.layers.13.self_attention.linear_qkv.weight module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.14.self_attention.k_layernorm.weight module.decoder.layers.14.mlp.router.weight module.decoder.layers.15.self_attention.linear_proj.weight - module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.13.self_attention.linear_proj.weight Params for bucket 16 (49031936 elements, 49031936 padded size): module.decoder.layers.12.mlp.router.weight module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.12.self_attention.q_layernorm.weight - module.decoder.layers.12.self_attention.k_layernorm.weight module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.11.mlp.router.weight module.decoder.layers.10.mlp.router.weight + module.decoder.layers.11.mlp.router.weight Params for bucket 17 (46672384 elements, 46672384 padded size): - module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.self_attention.linear_proj.weight Params for bucket 18 (49031936 elements, 49031936 padded size): module.decoder.layers.7.mlp.router.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.q_layernorm.weight - module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight Params for bucket 19 (46672384 elements, 46672384 padded size): - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.q_layernorm.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.4.self_attention.k_layernorm.weight - module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight @@ -978,26 +971,33 @@ Params for bucket 19 (46672384 elements, 46672384 padded size): module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 20 (49031936 elements, 49031936 padded size): - module.decoder.layers.0.self_attention.k_layernorm.weight - module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.1.self_attention.k_layernorm.weight module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight Params for bucket 21 (111413248 elements, 111413248 padded size): module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.0.self_attention.linear_proj.weight @@ -1044,32 +1044,24 @@ Params for bucket 2 (40894464 elements, 40894464 padded size): Params for bucket 3 (40894464 elements, 40894464 padded size): module.decoder.layers.46.mlp.experts.linear_fc2.weight4 module.decoder.layers.46.mlp.experts.linear_fc2.weight12 - module.decoder.layers.46.mlp.experts.linear_fc1.weight6 module.decoder.layers.46.mlp.experts.linear_fc2.weight9 - module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc1.weight4 module.decoder.layers.46.mlp.experts.linear_fc2.weight6 module.decoder.layers.46.mlp.experts.linear_fc2.weight14 - module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc1.weight1 module.decoder.layers.46.mlp.experts.linear_fc2.weight11 - module.decoder.layers.46.mlp.experts.linear_fc1.weight5 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 module.decoder.layers.46.mlp.experts.linear_fc2.weight8 - module.decoder.layers.46.mlp.experts.linear_fc1.weight2 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 module.decoder.layers.46.mlp.experts.linear_fc2.weight5 module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 module.decoder.layers.46.mlp.experts.linear_fc2.weight10 - module.decoder.layers.46.mlp.experts.linear_fc1.weight4 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 module.decoder.layers.46.mlp.experts.linear_fc2.weight7 module.decoder.layers.46.mlp.experts.linear_fc2.weight15 - module.decoder.layers.46.mlp.experts.linear_fc1.weight1 + module.decoder.layers.46.mlp.experts.linear_fc1.weight2 Params for bucket 4 (40894464 elements, 40894464 padded size): - module.decoder.layers.46.mlp.experts.linear_fc1.weight10 - module.decoder.layers.46.mlp.experts.linear_fc1.weight8 - module.decoder.layers.46.mlp.experts.linear_fc1.weight12 - module.decoder.layers.46.mlp.experts.linear_fc1.weight14 - module.decoder.layers.45.mlp.experts.linear_fc2.weight0 - module.decoder.layers.45.mlp.experts.linear_fc2.weight2 - module.decoder.layers.45.mlp.experts.linear_fc2.weight4 - module.decoder.layers.45.mlp.experts.linear_fc2.weight6 module.decoder.layers.46.mlp.experts.linear_fc1.weight7 module.decoder.layers.46.mlp.experts.linear_fc1.weight9 module.decoder.layers.46.mlp.experts.linear_fc1.weight11 @@ -1079,15 +1071,15 @@ Params for bucket 4 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc2.weight3 module.decoder.layers.45.mlp.experts.linear_fc2.weight5 module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 Params for bucket 5 (40894464 elements, 40894464 padded size): - module.decoder.layers.45.mlp.experts.linear_fc2.weight8 - module.decoder.layers.45.mlp.experts.linear_fc2.weight12 - module.decoder.layers.45.mlp.experts.linear_fc2.weight10 - module.decoder.layers.45.mlp.experts.linear_fc2.weight14 - module.decoder.layers.45.mlp.experts.linear_fc1.weight1 - module.decoder.layers.45.mlp.experts.linear_fc1.weight3 - module.decoder.layers.45.mlp.experts.linear_fc1.weight5 - module.decoder.layers.45.mlp.experts.linear_fc1.weight7 module.decoder.layers.45.mlp.experts.linear_fc2.weight11 module.decoder.layers.45.mlp.experts.linear_fc2.weight9 module.decoder.layers.45.mlp.experts.linear_fc2.weight13 @@ -1097,7 +1089,21 @@ Params for bucket 5 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc1.weight4 module.decoder.layers.45.mlp.experts.linear_fc1.weight6 module.decoder.layers.45.mlp.experts.linear_fc1.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight4 module.decoder.layers.45.mlp.experts.linear_fc1.weight15 module.decoder.layers.44.mlp.experts.linear_fc2.weight1 module.decoder.layers.44.mlp.experts.linear_fc2.weight9 @@ -1111,13 +1117,14 @@ Params for bucket 6 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc2.weight8 module.decoder.layers.45.mlp.experts.linear_fc1.weight11 module.decoder.layers.44.mlp.experts.linear_fc2.weight5 - module.decoder.layers.44.mlp.experts.linear_fc2.weight2 - module.decoder.layers.44.mlp.experts.linear_fc2.weight10 - module.decoder.layers.45.mlp.experts.linear_fc1.weight13 - module.decoder.layers.44.mlp.experts.linear_fc2.weight7 - module.decoder.layers.45.mlp.experts.linear_fc1.weight10 - module.decoder.layers.44.mlp.experts.linear_fc2.weight4 Params for bucket 7 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 module.decoder.layers.44.mlp.experts.linear_fc2.weight15 module.decoder.layers.44.mlp.experts.linear_fc1.weight0 module.decoder.layers.44.mlp.experts.linear_fc2.weight13 @@ -1126,14 +1133,15 @@ Params for bucket 7 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc1.weight6 module.decoder.layers.44.mlp.experts.linear_fc1.weight8 module.decoder.layers.44.mlp.experts.linear_fc1.weight10 - module.decoder.layers.44.mlp.experts.linear_fc2.weight12 - module.decoder.layers.44.mlp.experts.linear_fc2.weight14 - module.decoder.layers.44.mlp.experts.linear_fc1.weight1 - module.decoder.layers.44.mlp.experts.linear_fc1.weight3 - module.decoder.layers.44.mlp.experts.linear_fc1.weight5 - module.decoder.layers.44.mlp.experts.linear_fc1.weight7 - module.decoder.layers.44.mlp.experts.linear_fc1.weight9 Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 + module.decoder.layers.43.mlp.experts.linear_fc2.weight8 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight5 + module.decoder.layers.43.mlp.experts.linear_fc2.weight13 module.decoder.layers.43.mlp.experts.linear_fc2.weight2 module.decoder.layers.43.mlp.experts.linear_fc2.weight10 module.decoder.layers.44.mlp.experts.linear_fc1.weight13 @@ -1147,21 +1155,7 @@ Params for bucket 8 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc1.weight12 module.decoder.layers.43.mlp.experts.linear_fc2.weight6 module.decoder.layers.43.mlp.experts.linear_fc2.weight14 - module.decoder.layers.43.mlp.experts.linear_fc2.weight3 - module.decoder.layers.43.mlp.experts.linear_fc2.weight11 - module.decoder.layers.44.mlp.experts.linear_fc1.weight14 - module.decoder.layers.43.mlp.experts.linear_fc2.weight0 - module.decoder.layers.43.mlp.experts.linear_fc2.weight8 - module.decoder.layers.44.mlp.experts.linear_fc1.weight11 - module.decoder.layers.43.mlp.experts.linear_fc2.weight5 - module.decoder.layers.43.mlp.experts.linear_fc2.weight13 Params for bucket 9 (40894464 elements, 40894464 padded size): - module.decoder.layers.43.mlp.experts.linear_fc1.weight3 - module.decoder.layers.43.mlp.experts.linear_fc1.weight1 - module.decoder.layers.43.mlp.experts.linear_fc1.weight5 - module.decoder.layers.43.mlp.experts.linear_fc1.weight7 - module.decoder.layers.43.mlp.experts.linear_fc1.weight9 - module.decoder.layers.43.mlp.experts.linear_fc1.weight11 module.decoder.layers.43.mlp.experts.linear_fc1.weight0 module.decoder.layers.43.mlp.experts.linear_fc1.weight4 module.decoder.layers.43.mlp.experts.linear_fc1.weight2 @@ -1169,7 +1163,21 @@ Params for bucket 9 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc1.weight8 module.decoder.layers.43.mlp.experts.linear_fc1.weight10 module.decoder.layers.43.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 + module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 module.decoder.layers.43.mlp.experts.linear_fc1.weight13 module.decoder.layers.42.mlp.experts.linear_fc2.weight3 module.decoder.layers.42.mlp.experts.linear_fc2.weight11 @@ -1178,19 +1186,11 @@ Params for bucket 10 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc1.weight15 module.decoder.layers.42.mlp.experts.linear_fc2.weight5 module.decoder.layers.42.mlp.experts.linear_fc2.weight13 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight2 module.decoder.layers.42.mlp.experts.linear_fc2.weight10 module.decoder.layers.42.mlp.experts.linear_fc2.weight7 module.decoder.layers.42.mlp.experts.linear_fc2.weight15 - module.decoder.layers.42.mlp.experts.linear_fc1.weight1 - module.decoder.layers.43.mlp.experts.linear_fc1.weight14 - module.decoder.layers.42.mlp.experts.linear_fc2.weight4 - module.decoder.layers.42.mlp.experts.linear_fc2.weight12 - module.decoder.layers.42.mlp.experts.linear_fc2.weight1 - module.decoder.layers.42.mlp.experts.linear_fc2.weight9 - module.decoder.layers.42.mlp.experts.linear_fc2.weight6 - module.decoder.layers.42.mlp.experts.linear_fc2.weight14 - module.decoder.layers.42.mlp.experts.linear_fc1.weight0 Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight5 module.decoder.layers.42.mlp.experts.linear_fc1.weight3 @@ -1199,18 +1199,13 @@ Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight11 module.decoder.layers.42.mlp.experts.linear_fc1.weight13 module.decoder.layers.42.mlp.experts.linear_fc1.weight2 - module.decoder.layers.42.mlp.experts.linear_fc1.weight6 module.decoder.layers.42.mlp.experts.linear_fc1.weight4 + module.decoder.layers.42.mlp.experts.linear_fc1.weight6 module.decoder.layers.42.mlp.experts.linear_fc1.weight8 module.decoder.layers.42.mlp.experts.linear_fc1.weight10 module.decoder.layers.42.mlp.experts.linear_fc1.weight12 module.decoder.layers.42.mlp.experts.linear_fc1.weight14 Params for bucket 12 (40894464 elements, 40894464 padded size): - module.decoder.layers.41.mlp.experts.linear_fc2.weight3 - module.decoder.layers.41.mlp.experts.linear_fc2.weight11 - module.decoder.layers.41.mlp.experts.linear_fc2.weight0 - module.decoder.layers.41.mlp.experts.linear_fc2.weight8 - module.decoder.layers.41.mlp.experts.linear_fc1.weight3 module.decoder.layers.42.mlp.experts.linear_fc1.weight15 module.decoder.layers.41.mlp.experts.linear_fc2.weight5 module.decoder.layers.41.mlp.experts.linear_fc2.weight13 @@ -1227,6 +1222,11 @@ Params for bucket 12 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc2.weight6 module.decoder.layers.41.mlp.experts.linear_fc2.weight14 module.decoder.layers.41.mlp.experts.linear_fc1.weight1 + module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight11 + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 Params for bucket 13 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight4 module.decoder.layers.41.mlp.experts.linear_fc1.weight8 @@ -1235,18 +1235,14 @@ Params for bucket 13 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight12 module.decoder.layers.41.mlp.experts.linear_fc1.weight14 module.decoder.layers.40.mlp.experts.linear_fc2.weight0 - module.decoder.layers.41.mlp.experts.linear_fc1.weight5 module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 module.decoder.layers.41.mlp.experts.linear_fc1.weight9 module.decoder.layers.41.mlp.experts.linear_fc1.weight11 module.decoder.layers.41.mlp.experts.linear_fc1.weight13 module.decoder.layers.41.mlp.experts.linear_fc1.weight15 module.decoder.layers.40.mlp.experts.linear_fc2.weight1 Params for bucket 14 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc2.weight4 - module.decoder.layers.40.mlp.experts.linear_fc2.weight12 - module.decoder.layers.40.mlp.experts.linear_fc2.weight9 - module.decoder.layers.40.mlp.experts.linear_fc1.weight4 module.decoder.layers.40.mlp.experts.linear_fc2.weight6 module.decoder.layers.40.mlp.experts.linear_fc2.weight14 module.decoder.layers.40.mlp.experts.linear_fc1.weight1 @@ -1263,33 +1259,28 @@ Params for bucket 14 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc2.weight7 module.decoder.layers.40.mlp.experts.linear_fc2.weight15 module.decoder.layers.40.mlp.experts.linear_fc1.weight2 + module.decoder.layers.40.mlp.experts.linear_fc2.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight12 + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 Params for bucket 15 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc1.weight7 module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 module.decoder.layers.40.mlp.experts.linear_fc1.weight11 module.decoder.layers.40.mlp.experts.linear_fc1.weight13 module.decoder.layers.40.mlp.experts.linear_fc1.weight15 - module.decoder.layers.39.mlp.experts.linear_fc2.weight1 - module.decoder.layers.39.mlp.experts.linear_fc2.weight3 - module.decoder.layers.39.mlp.experts.linear_fc2.weight5 + module.decoder.layers.39.mlp.experts.linear_fc2.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight2 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 module.decoder.layers.40.mlp.experts.linear_fc1.weight6 - module.decoder.layers.40.mlp.experts.linear_fc1.weight8 module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 module.decoder.layers.40.mlp.experts.linear_fc1.weight12 module.decoder.layers.40.mlp.experts.linear_fc1.weight14 - module.decoder.layers.39.mlp.experts.linear_fc2.weight0 - module.decoder.layers.39.mlp.experts.linear_fc2.weight2 - module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc2.weight3 + module.decoder.layers.39.mlp.experts.linear_fc2.weight5 Params for bucket 16 (40894464 elements, 40894464 padded size): - module.decoder.layers.39.mlp.experts.linear_fc2.weight9 - module.decoder.layers.39.mlp.experts.linear_fc2.weight7 - module.decoder.layers.39.mlp.experts.linear_fc2.weight11 - module.decoder.layers.39.mlp.experts.linear_fc2.weight13 - module.decoder.layers.39.mlp.experts.linear_fc2.weight15 - module.decoder.layers.39.mlp.experts.linear_fc1.weight0 - module.decoder.layers.39.mlp.experts.linear_fc1.weight2 - module.decoder.layers.39.mlp.experts.linear_fc1.weight4 - module.decoder.layers.39.mlp.experts.linear_fc1.weight6 module.decoder.layers.39.mlp.experts.linear_fc2.weight6 module.decoder.layers.39.mlp.experts.linear_fc2.weight10 module.decoder.layers.39.mlp.experts.linear_fc2.weight8 @@ -1299,16 +1290,16 @@ Params for bucket 16 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc1.weight3 module.decoder.layers.39.mlp.experts.linear_fc1.weight5 module.decoder.layers.39.mlp.experts.linear_fc1.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.39.mlp.experts.linear_fc2.weight13 + module.decoder.layers.39.mlp.experts.linear_fc2.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc1.weight2 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 Params for bucket 17 (40894464 elements, 40894464 padded size): - module.decoder.layers.39.mlp.experts.linear_fc1.weight8 - module.decoder.layers.39.mlp.experts.linear_fc1.weight12 - module.decoder.layers.39.mlp.experts.linear_fc1.weight10 - module.decoder.layers.39.mlp.experts.linear_fc1.weight14 - module.decoder.layers.38.mlp.experts.linear_fc2.weight0 - module.decoder.layers.38.mlp.experts.linear_fc2.weight2 - module.decoder.layers.38.mlp.experts.linear_fc2.weight4 - module.decoder.layers.38.mlp.experts.linear_fc2.weight6 - module.decoder.layers.38.mlp.experts.linear_fc2.weight8 module.decoder.layers.39.mlp.experts.linear_fc1.weight11 module.decoder.layers.39.mlp.experts.linear_fc1.weight9 module.decoder.layers.39.mlp.experts.linear_fc1.weight13 @@ -1318,15 +1309,16 @@ Params for bucket 17 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc2.weight5 module.decoder.layers.38.mlp.experts.linear_fc2.weight7 module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 Params for bucket 18 (40894464 elements, 40894464 padded size): - module.decoder.layers.38.mlp.experts.linear_fc2.weight10 - module.decoder.layers.38.mlp.experts.linear_fc2.weight14 - module.decoder.layers.38.mlp.experts.linear_fc2.weight12 - module.decoder.layers.38.mlp.experts.linear_fc1.weight1 - module.decoder.layers.38.mlp.experts.linear_fc1.weight3 - module.decoder.layers.38.mlp.experts.linear_fc1.weight5 - module.decoder.layers.38.mlp.experts.linear_fc1.weight7 - module.decoder.layers.38.mlp.experts.linear_fc1.weight9 module.decoder.layers.38.mlp.experts.linear_fc2.weight13 module.decoder.layers.38.mlp.experts.linear_fc2.weight11 module.decoder.layers.38.mlp.experts.linear_fc2.weight15 @@ -1335,7 +1327,22 @@ Params for bucket 18 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc1.weight4 module.decoder.layers.38.mlp.experts.linear_fc1.weight6 module.decoder.layers.38.mlp.experts.linear_fc1.weight8 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight1 + module.decoder.layers.38.mlp.experts.linear_fc1.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight5 + module.decoder.layers.38.mlp.experts.linear_fc1.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight9 Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight14 + module.decoder.layers.37.mlp.experts.linear_fc2.weight4 + module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 module.decoder.layers.38.mlp.experts.linear_fc1.weight13 module.decoder.layers.37.mlp.experts.linear_fc2.weight3 module.decoder.layers.37.mlp.experts.linear_fc2.weight11 @@ -1349,21 +1356,7 @@ Params for bucket 19 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc2.weight2 module.decoder.layers.37.mlp.experts.linear_fc2.weight10 module.decoder.layers.37.mlp.experts.linear_fc2.weight7 - module.decoder.layers.38.mlp.experts.linear_fc1.weight14 - module.decoder.layers.37.mlp.experts.linear_fc2.weight4 - module.decoder.layers.37.mlp.experts.linear_fc2.weight12 - module.decoder.layers.38.mlp.experts.linear_fc1.weight11 - module.decoder.layers.37.mlp.experts.linear_fc2.weight1 - module.decoder.layers.37.mlp.experts.linear_fc2.weight9 - module.decoder.layers.37.mlp.experts.linear_fc2.weight6 Params for bucket 20 (40894464 elements, 40894464 padded size): - module.decoder.layers.37.mlp.experts.linear_fc1.weight2 - module.decoder.layers.37.mlp.experts.linear_fc2.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight0 - module.decoder.layers.37.mlp.experts.linear_fc1.weight4 - module.decoder.layers.37.mlp.experts.linear_fc1.weight6 - module.decoder.layers.37.mlp.experts.linear_fc1.weight8 - module.decoder.layers.37.mlp.experts.linear_fc1.weight10 module.decoder.layers.37.mlp.experts.linear_fc2.weight14 module.decoder.layers.37.mlp.experts.linear_fc1.weight1 module.decoder.layers.37.mlp.experts.linear_fc1.weight3 @@ -1371,7 +1364,23 @@ Params for bucket 20 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc1.weight7 module.decoder.layers.37.mlp.experts.linear_fc1.weight9 module.decoder.layers.37.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight2 + module.decoder.layers.36.mlp.experts.linear_fc2.weight10 + module.decoder.layers.36.mlp.experts.linear_fc2.weight7 + module.decoder.layers.36.mlp.experts.linear_fc2.weight15 module.decoder.layers.37.mlp.experts.linear_fc1.weight14 module.decoder.layers.36.mlp.experts.linear_fc2.weight4 module.decoder.layers.36.mlp.experts.linear_fc2.weight12 @@ -1384,16 +1393,13 @@ Params for bucket 21 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc2.weight11 module.decoder.layers.36.mlp.experts.linear_fc2.weight0 module.decoder.layers.36.mlp.experts.linear_fc2.weight8 - module.decoder.layers.37.mlp.experts.linear_fc1.weight15 - module.decoder.layers.36.mlp.experts.linear_fc2.weight5 - module.decoder.layers.36.mlp.experts.linear_fc2.weight13 - module.decoder.layers.36.mlp.experts.linear_fc1.weight0 - module.decoder.layers.37.mlp.experts.linear_fc1.weight12 - module.decoder.layers.36.mlp.experts.linear_fc2.weight2 - module.decoder.layers.36.mlp.experts.linear_fc2.weight10 - module.decoder.layers.36.mlp.experts.linear_fc2.weight7 - module.decoder.layers.36.mlp.experts.linear_fc2.weight15 Params for bucket 22 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 module.decoder.layers.36.mlp.experts.linear_fc1.weight1 module.decoder.layers.36.mlp.experts.linear_fc1.weight5 module.decoder.layers.36.mlp.experts.linear_fc1.weight3 @@ -1401,13 +1407,14 @@ Params for bucket 22 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc1.weight9 module.decoder.layers.36.mlp.experts.linear_fc1.weight11 module.decoder.layers.36.mlp.experts.linear_fc1.weight13 - module.decoder.layers.36.mlp.experts.linear_fc1.weight4 - module.decoder.layers.36.mlp.experts.linear_fc1.weight2 - module.decoder.layers.36.mlp.experts.linear_fc1.weight6 - module.decoder.layers.36.mlp.experts.linear_fc1.weight8 - module.decoder.layers.36.mlp.experts.linear_fc1.weight10 - module.decoder.layers.36.mlp.experts.linear_fc1.weight12 Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight3 + module.decoder.layers.35.mlp.experts.linear_fc2.weight11 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight8 module.decoder.layers.36.mlp.experts.linear_fc1.weight15 module.decoder.layers.35.mlp.experts.linear_fc2.weight5 module.decoder.layers.35.mlp.experts.linear_fc2.weight13 @@ -1422,20 +1429,7 @@ Params for bucket 23 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc2.weight12 module.decoder.layers.35.mlp.experts.linear_fc2.weight1 module.decoder.layers.35.mlp.experts.linear_fc2.weight9 - module.decoder.layers.35.mlp.experts.linear_fc2.weight6 - module.decoder.layers.35.mlp.experts.linear_fc2.weight14 - module.decoder.layers.35.mlp.experts.linear_fc1.weight1 - module.decoder.layers.35.mlp.experts.linear_fc2.weight3 - module.decoder.layers.35.mlp.experts.linear_fc2.weight11 - module.decoder.layers.35.mlp.experts.linear_fc2.weight0 - module.decoder.layers.35.mlp.experts.linear_fc2.weight8 Params for bucket 24 (40894464 elements, 40894464 padded size): - module.decoder.layers.35.mlp.experts.linear_fc1.weight6 - module.decoder.layers.35.mlp.experts.linear_fc1.weight4 - module.decoder.layers.35.mlp.experts.linear_fc1.weight8 - module.decoder.layers.35.mlp.experts.linear_fc1.weight10 - module.decoder.layers.35.mlp.experts.linear_fc1.weight12 - module.decoder.layers.35.mlp.experts.linear_fc1.weight14 module.decoder.layers.35.mlp.experts.linear_fc1.weight3 module.decoder.layers.35.mlp.experts.linear_fc1.weight7 module.decoder.layers.35.mlp.experts.linear_fc1.weight5 @@ -1443,7 +1437,21 @@ Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight11 module.decoder.layers.35.mlp.experts.linear_fc1.weight13 module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 module.decoder.layers.34.mlp.experts.linear_fc2.weight2 module.decoder.layers.34.mlp.experts.linear_fc2.weight10 module.decoder.layers.34.mlp.experts.linear_fc1.weight1 @@ -1457,31 +1465,31 @@ Params for bucket 25 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc1.weight0 module.decoder.layers.34.mlp.experts.linear_fc2.weight6 module.decoder.layers.34.mlp.experts.linear_fc2.weight14 - module.decoder.layers.34.mlp.experts.linear_fc2.weight3 - module.decoder.layers.34.mlp.experts.linear_fc2.weight11 - module.decoder.layers.34.mlp.experts.linear_fc1.weight2 - module.decoder.layers.34.mlp.experts.linear_fc2.weight0 - module.decoder.layers.34.mlp.experts.linear_fc2.weight8 - module.decoder.layers.34.mlp.experts.linear_fc2.weight5 - module.decoder.layers.34.mlp.experts.linear_fc2.weight13 - module.decoder.layers.34.mlp.experts.linear_fc1.weight4 Params for bucket 26 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc1.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight6 + module.decoder.layers.34.mlp.experts.linear_fc1.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight2 module.decoder.layers.34.mlp.experts.linear_fc1.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight9 module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 module.decoder.layers.34.mlp.experts.linear_fc1.weight11 module.decoder.layers.34.mlp.experts.linear_fc1.weight13 module.decoder.layers.34.mlp.experts.linear_fc1.weight15 module.decoder.layers.33.mlp.experts.linear_fc2.weight1 module.decoder.layers.33.mlp.experts.linear_fc2.weight3 - module.decoder.layers.34.mlp.experts.linear_fc1.weight6 - module.decoder.layers.34.mlp.experts.linear_fc1.weight8 - module.decoder.layers.34.mlp.experts.linear_fc1.weight10 - module.decoder.layers.34.mlp.experts.linear_fc1.weight12 - module.decoder.layers.34.mlp.experts.linear_fc1.weight14 - module.decoder.layers.33.mlp.experts.linear_fc2.weight0 - module.decoder.layers.33.mlp.experts.linear_fc2.weight2 Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 module.decoder.layers.33.mlp.experts.linear_fc2.weight11 module.decoder.layers.33.mlp.experts.linear_fc1.weight2 module.decoder.layers.33.mlp.experts.linear_fc2.weight8 @@ -1493,43 +1501,27 @@ Params for bucket 27 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc2.weight7 module.decoder.layers.33.mlp.experts.linear_fc2.weight15 module.decoder.layers.33.mlp.experts.linear_fc1.weight6 - module.decoder.layers.33.mlp.experts.linear_fc2.weight4 - module.decoder.layers.33.mlp.experts.linear_fc2.weight12 - module.decoder.layers.33.mlp.experts.linear_fc1.weight3 - module.decoder.layers.33.mlp.experts.linear_fc2.weight9 - module.decoder.layers.33.mlp.experts.linear_fc1.weight0 - module.decoder.layers.33.mlp.experts.linear_fc2.weight6 - module.decoder.layers.33.mlp.experts.linear_fc2.weight14 - module.decoder.layers.33.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40894464 elements, 40894464 padded size): - module.decoder.layers.33.mlp.experts.linear_fc1.weight10 - module.decoder.layers.33.mlp.experts.linear_fc1.weight8 - module.decoder.layers.33.mlp.experts.linear_fc1.weight12 - module.decoder.layers.33.mlp.experts.linear_fc1.weight14 - module.decoder.layers.32.mlp.experts.linear_fc2.weight0 - module.decoder.layers.32.mlp.experts.linear_fc2.weight2 - module.decoder.layers.32.mlp.experts.linear_fc2.weight4 - module.decoder.layers.32.mlp.experts.linear_fc2.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight7 - module.decoder.layers.33.mlp.experts.linear_fc1.weight9 module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight9 module.decoder.layers.33.mlp.experts.linear_fc1.weight13 module.decoder.layers.33.mlp.experts.linear_fc1.weight15 module.decoder.layers.32.mlp.experts.linear_fc2.weight1 module.decoder.layers.32.mlp.experts.linear_fc2.weight3 module.decoder.layers.32.mlp.experts.linear_fc2.weight5 module.decoder.layers.32.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight10 + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc2.weight0 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc2.weight6 Params for bucket 29 (40894464 elements, 40894464 padded size): - module.decoder.layers.32.mlp.experts.linear_fc2.weight8 - module.decoder.layers.32.mlp.experts.linear_fc2.weight10 - module.decoder.layers.32.mlp.experts.linear_fc2.weight12 - module.decoder.layers.32.mlp.experts.linear_fc2.weight14 - module.decoder.layers.32.mlp.experts.linear_fc1.weight1 - module.decoder.layers.32.mlp.experts.linear_fc1.weight3 - module.decoder.layers.32.mlp.experts.linear_fc1.weight5 - module.decoder.layers.32.mlp.experts.linear_fc1.weight7 - module.decoder.layers.32.mlp.experts.linear_fc2.weight11 module.decoder.layers.32.mlp.experts.linear_fc2.weight9 + module.decoder.layers.32.mlp.experts.linear_fc2.weight11 module.decoder.layers.32.mlp.experts.linear_fc2.weight13 module.decoder.layers.32.mlp.experts.linear_fc2.weight15 module.decoder.layers.32.mlp.experts.linear_fc1.weight0 @@ -1537,26 +1529,34 @@ Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight4 module.decoder.layers.32.mlp.experts.linear_fc1.weight6 module.decoder.layers.32.mlp.experts.linear_fc1.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 + module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight7 Params for bucket 30 (40894464 elements, 40894464 padded size): - module.decoder.layers.32.mlp.experts.linear_fc1.weight15 - module.decoder.layers.31.mlp.experts.linear_fc2.weight4 - module.decoder.layers.32.mlp.experts.linear_fc1.weight12 - module.decoder.layers.31.mlp.experts.linear_fc2.weight1 - module.decoder.layers.31.mlp.experts.linear_fc2.weight9 - module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.31.mlp.experts.linear_fc2.weight6 - module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.32.mlp.experts.linear_fc1.weight13 module.decoder.layers.31.mlp.experts.linear_fc2.weight3 module.decoder.layers.31.mlp.experts.linear_fc2.weight11 - module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 module.decoder.layers.31.mlp.experts.linear_fc2.weight0 module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 module.decoder.layers.31.mlp.experts.linear_fc2.weight5 - module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 module.decoder.layers.31.mlp.experts.linear_fc2.weight2 module.decoder.layers.31.mlp.experts.linear_fc2.weight10 - module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 Params for bucket 31 (40894464 elements, 40894464 padded size): module.decoder.layers.31.mlp.experts.linear_fc2.weight12 module.decoder.layers.31.mlp.experts.linear_fc2.weight14 @@ -1565,21 +1565,15 @@ Params for bucket 31 (40894464 elements, 40894464 padded size): module.decoder.layers.31.mlp.experts.linear_fc1.weight5 module.decoder.layers.31.mlp.experts.linear_fc1.weight7 module.decoder.layers.31.mlp.experts.linear_fc1.weight9 - module.decoder.layers.31.mlp.experts.linear_fc2.weight13 - module.decoder.layers.31.mlp.experts.linear_fc2.weight15 module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc1.weight2 module.decoder.layers.31.mlp.experts.linear_fc1.weight4 module.decoder.layers.31.mlp.experts.linear_fc1.weight6 module.decoder.layers.31.mlp.experts.linear_fc1.weight8 module.decoder.layers.31.mlp.experts.linear_fc1.weight10 Params for bucket 32 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc1.weight15 - module.decoder.layers.30.mlp.experts.linear_fc2.weight5 - module.decoder.layers.30.mlp.experts.linear_fc2.weight13 - module.decoder.layers.31.mlp.experts.linear_fc1.weight12 - module.decoder.layers.30.mlp.experts.linear_fc2.weight2 - module.decoder.layers.30.mlp.experts.linear_fc2.weight10 module.decoder.layers.30.mlp.experts.linear_fc2.weight7 module.decoder.layers.30.mlp.experts.linear_fc2.weight15 module.decoder.layers.31.mlp.experts.linear_fc1.weight14 @@ -1595,27 +1589,27 @@ Params for bucket 32 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc2.weight11 module.decoder.layers.30.mlp.experts.linear_fc2.weight0 module.decoder.layers.30.mlp.experts.linear_fc2.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.30.mlp.experts.linear_fc2.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 Params for bucket 33 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc1.weight0 - module.decoder.layers.30.mlp.experts.linear_fc1.weight2 module.decoder.layers.30.mlp.experts.linear_fc1.weight4 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 module.decoder.layers.30.mlp.experts.linear_fc1.weight6 module.decoder.layers.30.mlp.experts.linear_fc1.weight8 module.decoder.layers.30.mlp.experts.linear_fc1.weight10 module.decoder.layers.30.mlp.experts.linear_fc1.weight12 - module.decoder.layers.30.mlp.experts.linear_fc1.weight3 module.decoder.layers.30.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight3 module.decoder.layers.30.mlp.experts.linear_fc1.weight5 module.decoder.layers.30.mlp.experts.linear_fc1.weight7 module.decoder.layers.30.mlp.experts.linear_fc1.weight9 - module.decoder.layers.30.mlp.experts.linear_fc1.weight11 -Params for bucket 34 (40894464 elements, 40894464 padded size): - module.decoder.layers.29.mlp.experts.linear_fc2.weight2 - module.decoder.layers.29.mlp.experts.linear_fc2.weight10 - module.decoder.layers.29.mlp.experts.linear_fc1.weight1 - module.decoder.layers.30.mlp.experts.linear_fc1.weight13 - module.decoder.layers.29.mlp.experts.linear_fc2.weight7 - module.decoder.layers.29.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight11 +Params for bucket 34 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc2.weight4 module.decoder.layers.29.mlp.experts.linear_fc2.weight12 module.decoder.layers.29.mlp.experts.linear_fc2.weight1 @@ -1626,14 +1620,20 @@ Params for bucket 34 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc2.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight3 module.decoder.layers.29.mlp.experts.linear_fc2.weight11 - module.decoder.layers.29.mlp.experts.linear_fc2.weight0 module.decoder.layers.30.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 module.decoder.layers.29.mlp.experts.linear_fc2.weight8 module.decoder.layers.29.mlp.experts.linear_fc2.weight5 module.decoder.layers.29.mlp.experts.linear_fc2.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight2 + module.decoder.layers.29.mlp.experts.linear_fc2.weight10 + module.decoder.layers.29.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc1.weight13 + module.decoder.layers.29.mlp.experts.linear_fc2.weight7 + module.decoder.layers.29.mlp.experts.linear_fc2.weight15 Params for bucket 35 (40894464 elements, 40894464 padded size): - module.decoder.layers.29.mlp.experts.linear_fc1.weight5 module.decoder.layers.29.mlp.experts.linear_fc1.weight3 + module.decoder.layers.29.mlp.experts.linear_fc1.weight5 module.decoder.layers.29.mlp.experts.linear_fc1.weight7 module.decoder.layers.29.mlp.experts.linear_fc1.weight9 module.decoder.layers.29.mlp.experts.linear_fc1.weight11 @@ -1646,11 +1646,6 @@ Params for bucket 35 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc1.weight12 module.decoder.layers.29.mlp.experts.linear_fc1.weight14 Params for bucket 36 (40894464 elements, 40894464 padded size): - module.decoder.layers.28.mlp.experts.linear_fc2.weight3 - module.decoder.layers.28.mlp.experts.linear_fc2.weight11 - module.decoder.layers.28.mlp.experts.linear_fc1.weight2 - module.decoder.layers.28.mlp.experts.linear_fc2.weight0 - module.decoder.layers.28.mlp.experts.linear_fc2.weight8 module.decoder.layers.28.mlp.experts.linear_fc2.weight5 module.decoder.layers.28.mlp.experts.linear_fc2.weight13 module.decoder.layers.28.mlp.experts.linear_fc2.weight2 @@ -1667,10 +1662,15 @@ Params for bucket 36 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight6 module.decoder.layers.28.mlp.experts.linear_fc2.weight14 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc2.weight8 Params for bucket 37 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc1.weight4 - module.decoder.layers.28.mlp.experts.linear_fc1.weight8 module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 module.decoder.layers.28.mlp.experts.linear_fc1.weight10 module.decoder.layers.28.mlp.experts.linear_fc1.weight12 module.decoder.layers.28.mlp.experts.linear_fc1.weight14 @@ -1683,16 +1683,6 @@ Params for bucket 37 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc1.weight15 module.decoder.layers.27.mlp.experts.linear_fc2.weight1 Params for bucket 38 (40894464 elements, 40894464 padded size): - module.decoder.layers.27.mlp.experts.linear_fc2.weight2 - module.decoder.layers.27.mlp.experts.linear_fc2.weight10 - module.decoder.layers.27.mlp.experts.linear_fc1.weight5 - module.decoder.layers.27.mlp.experts.linear_fc2.weight7 - module.decoder.layers.27.mlp.experts.linear_fc2.weight15 - module.decoder.layers.27.mlp.experts.linear_fc1.weight2 - module.decoder.layers.27.mlp.experts.linear_fc2.weight4 - module.decoder.layers.27.mlp.experts.linear_fc2.weight12 - module.decoder.layers.27.mlp.experts.linear_fc2.weight9 - module.decoder.layers.27.mlp.experts.linear_fc1.weight4 module.decoder.layers.27.mlp.experts.linear_fc2.weight6 module.decoder.layers.27.mlp.experts.linear_fc2.weight14 module.decoder.layers.27.mlp.experts.linear_fc1.weight1 @@ -1703,6 +1693,16 @@ Params for bucket 38 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc2.weight5 module.decoder.layers.27.mlp.experts.linear_fc2.weight13 module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight10 + module.decoder.layers.27.mlp.experts.linear_fc1.weight5 + module.decoder.layers.27.mlp.experts.linear_fc2.weight7 + module.decoder.layers.27.mlp.experts.linear_fc2.weight15 + module.decoder.layers.27.mlp.experts.linear_fc1.weight2 + module.decoder.layers.27.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc2.weight12 + module.decoder.layers.27.mlp.experts.linear_fc2.weight9 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc1.weight7 module.decoder.layers.27.mlp.experts.linear_fc1.weight9 @@ -1713,8 +1713,8 @@ Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc2.weight3 module.decoder.layers.26.mlp.experts.linear_fc2.weight5 module.decoder.layers.27.mlp.experts.linear_fc1.weight6 - module.decoder.layers.27.mlp.experts.linear_fc1.weight10 module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 module.decoder.layers.27.mlp.experts.linear_fc1.weight12 module.decoder.layers.27.mlp.experts.linear_fc1.weight14 module.decoder.layers.26.mlp.experts.linear_fc2.weight0 @@ -1731,8 +1731,8 @@ Params for bucket 40 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc1.weight4 module.decoder.layers.26.mlp.experts.linear_fc1.weight6 module.decoder.layers.26.mlp.experts.linear_fc2.weight6 - module.decoder.layers.26.mlp.experts.linear_fc2.weight10 module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 module.decoder.layers.26.mlp.experts.linear_fc2.weight12 module.decoder.layers.26.mlp.experts.linear_fc2.weight14 module.decoder.layers.26.mlp.experts.linear_fc1.weight1 @@ -1741,32 +1741,24 @@ Params for bucket 40 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc1.weight7 Params for bucket 41 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc1.weight8 - module.decoder.layers.26.mlp.experts.linear_fc1.weight12 module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 module.decoder.layers.26.mlp.experts.linear_fc1.weight14 - module.decoder.layers.25.mlp.experts.linear_fc2.weight0 - module.decoder.layers.25.mlp.experts.linear_fc2.weight2 - module.decoder.layers.25.mlp.experts.linear_fc2.weight4 - module.decoder.layers.25.mlp.experts.linear_fc2.weight6 - module.decoder.layers.25.mlp.experts.linear_fc2.weight8 - module.decoder.layers.26.mlp.experts.linear_fc1.weight11 - module.decoder.layers.26.mlp.experts.linear_fc1.weight9 - module.decoder.layers.26.mlp.experts.linear_fc1.weight13 - module.decoder.layers.26.mlp.experts.linear_fc1.weight15 module.decoder.layers.25.mlp.experts.linear_fc2.weight1 module.decoder.layers.25.mlp.experts.linear_fc2.weight3 module.decoder.layers.25.mlp.experts.linear_fc2.weight5 module.decoder.layers.25.mlp.experts.linear_fc2.weight7 module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 Params for bucket 42 (40894464 elements, 40894464 padded size): - module.decoder.layers.25.mlp.experts.linear_fc2.weight10 - module.decoder.layers.25.mlp.experts.linear_fc2.weight12 - module.decoder.layers.25.mlp.experts.linear_fc2.weight14 - module.decoder.layers.25.mlp.experts.linear_fc1.weight1 - module.decoder.layers.25.mlp.experts.linear_fc1.weight3 - module.decoder.layers.25.mlp.experts.linear_fc1.weight5 - module.decoder.layers.25.mlp.experts.linear_fc1.weight7 - module.decoder.layers.25.mlp.experts.linear_fc1.weight9 module.decoder.layers.25.mlp.experts.linear_fc2.weight13 module.decoder.layers.25.mlp.experts.linear_fc2.weight11 module.decoder.layers.25.mlp.experts.linear_fc2.weight15 @@ -1775,35 +1767,36 @@ Params for bucket 42 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc1.weight4 module.decoder.layers.25.mlp.experts.linear_fc1.weight6 module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight12 module.decoder.layers.25.mlp.experts.linear_fc1.weight15 module.decoder.layers.24.mlp.experts.linear_fc2.weight9 module.decoder.layers.25.mlp.experts.linear_fc1.weight12 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 module.decoder.layers.24.mlp.experts.linear_fc2.weight6 - module.decoder.layers.24.mlp.experts.linear_fc2.weight3 module.decoder.layers.24.mlp.experts.linear_fc2.weight11 module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 module.decoder.layers.24.mlp.experts.linear_fc2.weight8 module.decoder.layers.25.mlp.experts.linear_fc1.weight11 module.decoder.layers.24.mlp.experts.linear_fc2.weight1 module.decoder.layers.24.mlp.experts.linear_fc2.weight5 module.decoder.layers.24.mlp.experts.linear_fc2.weight13 - module.decoder.layers.24.mlp.experts.linear_fc2.weight2 module.decoder.layers.24.mlp.experts.linear_fc2.weight10 module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 module.decoder.layers.24.mlp.experts.linear_fc2.weight7 - module.decoder.layers.25.mlp.experts.linear_fc1.weight10 - module.decoder.layers.24.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc2.weight4 - module.decoder.layers.24.mlp.experts.linear_fc2.weight12 Params for bucket 44 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc1.weight2 - module.decoder.layers.24.mlp.experts.linear_fc2.weight15 - module.decoder.layers.24.mlp.experts.linear_fc1.weight0 - module.decoder.layers.24.mlp.experts.linear_fc1.weight4 - module.decoder.layers.24.mlp.experts.linear_fc1.weight6 - module.decoder.layers.24.mlp.experts.linear_fc1.weight8 - module.decoder.layers.24.mlp.experts.linear_fc1.weight10 module.decoder.layers.24.mlp.experts.linear_fc2.weight14 module.decoder.layers.24.mlp.experts.linear_fc1.weight1 module.decoder.layers.24.mlp.experts.linear_fc1.weight3 @@ -1811,7 +1804,16 @@ Params for bucket 44 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc1.weight7 module.decoder.layers.24.mlp.experts.linear_fc1.weight9 module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc1.weight6 + module.decoder.layers.24.mlp.experts.linear_fc1.weight8 + module.decoder.layers.24.mlp.experts.linear_fc1.weight10 Params for bucket 45 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight13 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight10 module.decoder.layers.24.mlp.experts.linear_fc1.weight13 @@ -1831,9 +1833,13 @@ Params for bucket 45 (40894464 elements, 40894464 padded size): module.decoder.layers.23.mlp.experts.linear_fc2.weight0 module.decoder.layers.24.mlp.experts.linear_fc1.weight14 module.decoder.layers.23.mlp.experts.linear_fc2.weight8 - module.decoder.layers.23.mlp.experts.linear_fc2.weight5 - module.decoder.layers.23.mlp.experts.linear_fc2.weight13 Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc1.weight8 + module.decoder.layers.23.mlp.experts.linear_fc1.weight10 + module.decoder.layers.23.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.23.mlp.experts.linear_fc1.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 @@ -1841,34 +1847,28 @@ Params for bucket 46 (40894464 elements, 40894464 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight9 module.decoder.layers.23.mlp.experts.linear_fc1.weight11 module.decoder.layers.23.mlp.experts.linear_fc1.weight13 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc1.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 - module.decoder.layers.23.mlp.experts.linear_fc1.weight8 - module.decoder.layers.23.mlp.experts.linear_fc1.weight10 - module.decoder.layers.23.mlp.experts.linear_fc1.weight12 Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight14 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight11 module.decoder.layers.23.mlp.experts.linear_fc1.weight14 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight5 module.decoder.layers.22.mlp.experts.linear_fc2.weight13 - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight10 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight15 - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 module.decoder.layers.22.mlp.experts.linear_fc2.weight12 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight9 - module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc2.weight14 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 Params for bucket 48 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 @@ -1877,18 +1877,13 @@ Params for bucket 48 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight12 module.decoder.layers.22.mlp.experts.linear_fc1.weight14 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight9 module.decoder.layers.22.mlp.experts.linear_fc1.weight11 module.decoder.layers.22.mlp.experts.linear_fc1.weight13 module.decoder.layers.22.mlp.experts.linear_fc1.weight15 Params for bucket 49 (40894464 elements, 40894464 padded size): - module.decoder.layers.21.mlp.experts.linear_fc2.weight5 - module.decoder.layers.21.mlp.experts.linear_fc2.weight13 - module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.21.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc2.weight10 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight15 module.decoder.layers.21.mlp.experts.linear_fc1.weight2 @@ -1905,6 +1900,11 @@ Params for bucket 49 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc2.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight13 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 Params for bucket 50 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight6 module.decoder.layers.21.mlp.experts.linear_fc1.weight8 @@ -1914,19 +1914,14 @@ Params for bucket 50 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.21.mlp.experts.linear_fc1.weight5 - module.decoder.layers.21.mlp.experts.linear_fc1.weight7 module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 module.decoder.layers.21.mlp.experts.linear_fc1.weight11 module.decoder.layers.21.mlp.experts.linear_fc1.weight13 module.decoder.layers.21.mlp.experts.linear_fc1.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 Params for bucket 51 (40894464 elements, 40894464 padded size): - module.decoder.layers.20.mlp.experts.linear_fc2.weight7 - module.decoder.layers.20.mlp.experts.linear_fc2.weight15 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc2.weight12 module.decoder.layers.20.mlp.experts.linear_fc2.weight9 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 @@ -1941,6 +1936,11 @@ Params for bucket 51 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight10 module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight10 module.decoder.layers.20.mlp.experts.linear_fc1.weight8 @@ -1951,8 +1951,8 @@ Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight11 module.decoder.layers.20.mlp.experts.linear_fc1.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 module.decoder.layers.20.mlp.experts.linear_fc1.weight13 module.decoder.layers.20.mlp.experts.linear_fc1.weight15 module.decoder.layers.19.mlp.experts.linear_fc2.weight1 @@ -1964,20 +1964,22 @@ Params for bucket 53 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight10 module.decoder.layers.19.mlp.experts.linear_fc2.weight12 module.decoder.layers.19.mlp.experts.linear_fc2.weight14 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight8 - module.decoder.layers.19.mlp.experts.linear_fc2.weight11 - module.decoder.layers.19.mlp.experts.linear_fc2.weight9 - module.decoder.layers.19.mlp.experts.linear_fc2.weight13 - module.decoder.layers.19.mlp.experts.linear_fc2.weight15 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 module.decoder.layers.19.mlp.experts.linear_fc1.weight10 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight15 @@ -1995,25 +1997,26 @@ Params for bucket 54 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc1.weight13 - module.decoder.layers.18.mlp.experts.linear_fc2.weight7 Params for bucket 55 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight12 - module.decoder.layers.18.mlp.experts.linear_fc2.weight14 - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.18.mlp.experts.linear_fc1.weight3 - module.decoder.layers.18.mlp.experts.linear_fc1.weight5 - module.decoder.layers.18.mlp.experts.linear_fc1.weight7 - module.decoder.layers.18.mlp.experts.linear_fc1.weight9 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 module.decoder.layers.18.mlp.experts.linear_fc2.weight15 module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.18.mlp.experts.linear_fc2.weight13 module.decoder.layers.18.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight8 module.decoder.layers.18.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight9 Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 module.decoder.layers.18.mlp.experts.linear_fc1.weight11 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight13 @@ -2032,10 +2035,13 @@ Params for bucket 56 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc2.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight11 - module.decoder.layers.18.mlp.experts.linear_fc1.weight14 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight8 Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight2 @@ -2043,13 +2049,9 @@ Params for bucket 57 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight8 module.decoder.layers.17.mlp.experts.linear_fc1.weight10 module.decoder.layers.17.mlp.experts.linear_fc1.weight12 - module.decoder.layers.17.mlp.experts.linear_fc1.weight3 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc1.weight9 - module.decoder.layers.17.mlp.experts.linear_fc1.weight11 Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 module.decoder.layers.16.mlp.experts.linear_fc2.weight14 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 @@ -2069,23 +2071,23 @@ Params for bucket 58 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight14 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight12 - module.decoder.layers.16.mlp.experts.linear_fc2.weight1 - module.decoder.layers.16.mlp.experts.linear_fc2.weight9 Params for bucket 59 (40894464 elements, 40894464 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight7 - module.decoder.layers.16.mlp.experts.linear_fc1.weight9 - module.decoder.layers.16.mlp.experts.linear_fc1.weight11 - module.decoder.layers.16.mlp.experts.linear_fc1.weight13 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 module.decoder.layers.16.mlp.experts.linear_fc1.weight8 module.decoder.layers.16.mlp.experts.linear_fc1.weight10 module.decoder.layers.16.mlp.experts.linear_fc1.weight12 module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight15 module.decoder.layers.15.mlp.experts.linear_fc1.weight2 @@ -2105,29 +2107,22 @@ Params for bucket 60 (40894464 elements, 40894464 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight5 module.decoder.layers.15.mlp.experts.linear_fc2.weight13 module.decoder.layers.15.mlp.experts.linear_fc1.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight10 Params for bucket 61 (40894464 elements, 40894464 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight4 - module.decoder.layers.15.mlp.experts.linear_fc1.weight8 - module.decoder.layers.15.mlp.experts.linear_fc1.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight10 - module.decoder.layers.15.mlp.experts.linear_fc1.weight12 - module.decoder.layers.15.mlp.experts.linear_fc1.weight14 - module.decoder.layers.14.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight9 module.decoder.layers.15.mlp.experts.linear_fc1.weight11 module.decoder.layers.15.mlp.experts.linear_fc1.weight13 module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 module.decoder.layers.14.mlp.experts.linear_fc2.weight1 Params for bucket 62 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight8 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight5 - module.decoder.layers.14.mlp.experts.linear_fc2.weight13 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 module.decoder.layers.14.mlp.experts.linear_fc2.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight10 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 @@ -2143,9 +2138,14 @@ Params for bucket 62 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 Params for bucket 63 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight11 module.decoder.layers.14.mlp.experts.linear_fc1.weight13 module.decoder.layers.14.mlp.experts.linear_fc1.weight15 @@ -2153,16 +2153,16 @@ Params for bucket 63 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight3 module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight6 - module.decoder.layers.14.mlp.experts.linear_fc1.weight8 module.decoder.layers.14.mlp.experts.linear_fc1.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 module.decoder.layers.14.mlp.experts.linear_fc1.weight12 module.decoder.layers.14.mlp.experts.linear_fc1.weight14 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight4 Params for bucket 64 (40894464 elements, 40894464 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight11 module.decoder.layers.13.mlp.experts.linear_fc2.weight13 module.decoder.layers.13.mlp.experts.linear_fc2.weight15 @@ -2189,8 +2189,8 @@ Params for bucket 65 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc2.weight4 module.decoder.layers.12.mlp.experts.linear_fc2.weight6 module.decoder.layers.12.mlp.experts.linear_fc2.weight8 - module.decoder.layers.13.mlp.experts.linear_fc1.weight11 module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 module.decoder.layers.13.mlp.experts.linear_fc1.weight13 module.decoder.layers.13.mlp.experts.linear_fc1.weight15 module.decoder.layers.12.mlp.experts.linear_fc2.weight1 @@ -2238,8 +2238,8 @@ Params for bucket 67 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight10 Params for bucket 68 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight8 @@ -2292,84 +2292,76 @@ Params for bucket 71 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight9 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 Params for bucket 72 (40894464 elements, 40894464 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight8 - module.decoder.layers.9.mlp.experts.linear_fc1.weight10 - module.decoder.layers.9.mlp.experts.linear_fc1.weight12 - module.decoder.layers.9.mlp.experts.linear_fc1.weight14 Params for bucket 73 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight12 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 - module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight13 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 Params for bucket 74 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight12 - module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 Params for bucket 75 (40894464 elements, 40894464 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight11 - module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 @@ -2381,52 +2373,51 @@ Params for bucket 75 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 76 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 Params for bucket 77 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight9 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 78 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight11 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight13 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight15 @@ -2437,6 +2428,15 @@ Params for bucket 78 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight9 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 @@ -2445,36 +2445,36 @@ Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight9 - module.decoder.layers.5.mlp.experts.linear_fc2.weight13 - module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight10 Params for bucket 80 (40894464 elements, 40894464 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight9 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 - module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight13 - module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 Params for bucket 81 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 @@ -2483,53 +2483,49 @@ Params for bucket 81 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight9 module.decoder.layers.4.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 Params for bucket 82 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 Params for bucket 83 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc1.weight9 - module.decoder.layers.3.mlp.experts.linear_fc1.weight11 - module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 module.decoder.layers.3.mlp.experts.linear_fc1.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 Params for bucket 84 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 @@ -2547,22 +2543,31 @@ Params for bucket 84 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 Params for bucket 85 (40894464 elements, 40894464 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 - module.decoder.layers.2.mlp.experts.linear_fc1.weight8 - module.decoder.layers.2.mlp.experts.linear_fc1.weight10 - module.decoder.layers.2.mlp.experts.linear_fc1.weight12 - module.decoder.layers.2.mlp.experts.linear_fc1.weight14 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 @@ -2578,59 +2583,54 @@ Params for bucket 86 (40894464 elements, 40894464 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight15 - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight12 Params for bucket 87 (40894464 elements, 40894464 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 Params for bucket 88 (40894464 elements, 40894464 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 Params for bucket 89 (25165824 elements, 25165824 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight8 - module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:49:20 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 01:35:30 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -2638,7 +2638,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -2651,14 +2651,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:49:20 +[after dataloaders are built] datetime: 2026-08-30 01:35:30 done with setup ... training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (5415.57, 5428.44) - train/valid/test-data-iterators-setup ..........: (65.19, 82.38) -[before the start of training step] datetime: 2026-08-29 23:49:20 + model-and-optimizer-setup ......................: (5378.68, 5388.06) + train/valid/test-data-iterators-setup ..........: (66.56, 73.70) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 01:35:30 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 sm_count : num_sms=32, num_sms_dedup=148 @@ -2670,36 +2670,37 @@ INFO:moonep.api:MoonEP comm buffer sizes: [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1280, 1024, 256, 640, 512, 3328, 256, 1408, 896, 1408, 128, 1152, 3712, 1024, 1792, 12544, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [123, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1024, 128, 896, 128, 640, 0, 768, 128, 128, 256, 256, 256, 384, 128, 3584, 1152, 23424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[66, 74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5248, 2176, 1792, 4224, 128, 384, 128, 128, 1280, 640, 2816, 3200, 128, 128, 4224, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, 80, 89, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] - [2026-08-29 23:49:42] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22687.1 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.354 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 +Number of parameters in transformer block in billions: 29.90 + [2026-08-30 01:35:53] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22683.0 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.354 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in embedding layers in billions: 0.21 -Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 30.10 Number of parameters in most loaded shard in billions: 30.1034 Theoretical memory footprints: weight and optimizer=215316.59 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 66603.12646484375 | max allocated: 66603.126953125 | reserved: 68288.0 | max reserved: 68288.0 - [2026-08-29 23:49:43] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 797.4 | throughput per GPU (TFLOP/s/GPU): 111.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.501 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:44] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 626.1 | throughput per GPU (TFLOP/s/GPU): 142.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008754E+01 | loss scale: 1.0 | grad norm: 180.880 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:54] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 735.5 | throughput per GPU (TFLOP/s/GPU): 121.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.503 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:54] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 642.2 | throughput per GPU (TFLOP/s/GPU): 138.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008852E+01 | loss scale: 1.0 | grad norm: 181.212 | number of skipped iterations: 0 | number of nan iterations: 0 | Capture CUDA graph for training!!! CUDA graph capture done!!! - [2026-08-29 23:49:45] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1651.9 | throughput per GPU (TFLOP/s/GPU): 53.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604504E+00 | loss scale: 1.0 | grad norm: 8.724 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:46] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 360.4 | throughput per GPU (TFLOP/s/GPU): 247.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307546E+00 | loss scale: 1.0 | grad norm: 4.226 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:46] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 361.0 | throughput per GPU (TFLOP/s/GPU): 246.9 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.275688E+00 | loss scale: 1.0 | grad norm: 7.280 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:47] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 371.2 | throughput per GPU (TFLOP/s/GPU): 240.1 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.102283E+00 | loss scale: 1.0 | grad norm: 3.610 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:47] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 359.0 | throughput per GPU (TFLOP/s/GPU): 248.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.906293E+00 | loss scale: 1.0 | grad norm: 3.380 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:47] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 358.7 | throughput per GPU (TFLOP/s/GPU): 248.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.845001E+00 | loss scale: 1.0 | grad norm: 3.213 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:48] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 359.4 | throughput per GPU (TFLOP/s/GPU): 248.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.803046E+00 | loss scale: 1.0 | grad norm: 3.233 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:48] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 362.2 | throughput per GPU (TFLOP/s/GPU): 246.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.792130E+00 | loss scale: 1.0 | grad norm: 3.098 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:49:48] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 359.2 | throughput per GPU (TFLOP/s/GPU): 248.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.761202E+00 | loss scale: 1.0 | grad norm: 2.984 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:49:48 -W0829 23:49:49.289000 245313 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:49:49.290000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245499 closing signal SIGTERM -W0829 23:49:49.291000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245500 closing signal SIGTERM -W0829 23:49:49.292000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245501 closing signal SIGTERM -W0829 23:49:49.293000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245502 closing signal SIGTERM -W0829 23:49:49.294000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245503 closing signal SIGTERM -W0829 23:49:49.296000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245504 closing signal SIGTERM -W0829 23:49:49.296000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245505 closing signal SIGTERM -W0829 23:49:49.297000 245313 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 245506 closing signal SIGTERM + [2026-08-30 01:35:56] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1582.6 | throughput per GPU (TFLOP/s/GPU): 56.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604378E+00 | loss scale: 1.0 | grad norm: 8.711 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:56] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 356.8 | throughput per GPU (TFLOP/s/GPU): 249.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307677E+00 | loss scale: 1.0 | grad norm: 7.890 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:56] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 353.5 | throughput per GPU (TFLOP/s/GPU): 252.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.131496E+00 | loss scale: 1.0 | grad norm: 2.877 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:57] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 353.1 | throughput per GPU (TFLOP/s/GPU): 252.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.973051E+00 | loss scale: 1.0 | grad norm: 3.040 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:57] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 352.3 | throughput per GPU (TFLOP/s/GPU): 253.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.831199E+00 | loss scale: 1.0 | grad norm: 2.775 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:58] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 355.9 | throughput per GPU (TFLOP/s/GPU): 250.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.788062E+00 | loss scale: 1.0 | grad norm: 2.491 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:58] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 354.4 | throughput per GPU (TFLOP/s/GPU): 251.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.747782E+00 | loss scale: 1.0 | grad norm: 2.464 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:58] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 355.5 | throughput per GPU (TFLOP/s/GPU): 250.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.742904E+00 | loss scale: 1.0 | grad norm: 2.365 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:35:59] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 356.2 | throughput per GPU (TFLOP/s/GPU): 250.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.722297E+00 | loss scale: 1.0 | grad norm: 2.327 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 01:35:59 +[rank0]:[W830 01:36:00.573647244 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 01:36:01.028000 387880 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 01:36:01.029000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388067 closing signal SIGTERM +W0830 01:36:01.029000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388068 closing signal SIGTERM +W0830 01:36:01.031000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388069 closing signal SIGTERM +W0830 01:36:01.033000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388070 closing signal SIGTERM +W0830 01:36:01.034000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388071 closing signal SIGTERM +W0830 01:36:01.034000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388072 closing signal SIGTERM +W0830 01:36:01.035000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388073 closing signal SIGTERM +W0830 01:36:01.036000 387880 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 388074 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -2727,4 +2728,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 245313 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 387880 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/mem_samples.csv index e2bfbda5243f2ef182aa5c3c04852de0b5052cd2..d15853032de1728fb304c5d7b894754b6cefccff 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/mem_samples.csv @@ -26,8 +26,8 @@ 1, 1 2, 1 3, 1 -4, 4 -5, 4 +4, 1 +5, 3 6, 4 7, 4 0, 0 @@ -38,7 +38,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 4 1, 1 2, 1 3, 1 @@ -71,37 +71,37 @@ 6, 4 7, 4 0, 821 -1, 821 -2, 821 +1, 819 +2, 819 3, 821 -4, 821 -5, 821 -6, 821 -7, 821 -0, 1831 -1, 1831 -2, 1831 -3, 1831 -4, 1831 -5, 1831 -6, 1831 -7, 1831 -0, 4865 -1, 4629 -2, 4609 -3, 4809 -4, 4549 -5, 4965 -6, 4749 -7, 4689 -0, 46757 -1, 46765 -2, 46755 -3, 46739 -4, 46767 -5, 46729 -6, 46751 -7, 46785 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 46653 +1, 46653 +2, 46645 +3, 46629 +4, 46653 +5, 46657 +6, 46665 +7, 46651 0, 47547 1, 47549 2, 47549 @@ -110,54 +110,54 @@ 5, 47549 6, 47549 7, 47549 -0, 48081 -1, 47875 -2, 48085 -3, 47895 -4, 47895 -5, 48117 -6, 47927 -7, 48115 -0, 48651 -1, 48311 -2, 48695 -3, 48307 -4, 48475 -5, 48693 -6, 48343 -7, 48723 -0, 49249 +0, 48127 +1, 47943 +2, 48147 +3, 47929 +4, 47959 +5, 48155 +6, 47929 +7, 48161 +0, 48611 +1, 48293 +2, 48643 +3, 48267 +4, 48369 +5, 48631 +6, 48267 +7, 48655 +0, 49251 1, 48905 2, 49279 -3, 48907 +3, 48905 4, 49041 5, 49277 6, 48905 -7, 49279 +7, 49277 0, 49251 -1, 48905 +1, 48907 2, 49279 3, 48907 4, 49043 5, 49279 6, 48907 7, 49279 -0, 59887 -1, 59871 -2, 59861 -3, 59851 -4, 59879 -5, 59861 -6, 59851 -7, 59861 -0, 83441 -1, 83427 -2, 83429 -3, 83449 -4, 83467 -5, 83461 -6, 83421 -7, 83451 +0, 61515 +1, 61473 +2, 61549 +3, 61647 +4, 61675 +5, 61549 +6, 61519 +7, 61657 +0, 83447 +1, 83453 +2, 83441 +3, 83439 +4, 83495 +5, 83441 +6, 83447 +7, 83459 0, 83947 1, 83951 2, 83941 @@ -166,67 +166,67 @@ 5, 83941 6, 83931 7, 83941 -0, 84279 -1, 84219 -2, 84293 -3, 84269 -4, 84339 -5, 84307 -6, 84321 -7, 84273 -0, 84997 -1, 85001 -2, 84991 -3, 84981 -4, 85009 -5, 84991 -6, 84981 -7, 84991 -0, 95579 -1, 95473 -2, 95577 -3, 95317 -4, 95471 -5, 95567 -6, 95411 -7, 95607 -0, 103699 -1, 100805 -2, 102497 -3, 105057 -4, 102123 -5, 102643 -6, 101297 -7, 107699 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116555 -7, 116701 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116555 -7, 116701 -0, 23094 -1, 116609 -2, 116699 -3, 116457 -4, 116581 -5, 116699 -6, 116553 -7, 116699 -0, 22771 -1, 21174 -2, 21174 -3, 21174 -4, 21174 -5, 21174 -6, 21494 -7, 116699 +0, 84243 +1, 84235 +2, 84193 +3, 84217 +4, 84255 +5, 84271 +6, 84265 +7, 84283 +0, 90373 +1, 90401 +2, 91611 +3, 90369 +4, 91097 +5, 90391 +6, 90001 +7, 90243 +0, 95551 +1, 95415 +2, 95557 +3, 95319 +4, 95455 +5, 95591 +6, 95427 +7, 95569 +0, 102643 +1, 102153 +2, 103697 +3, 102399 +4, 102125 +5, 102751 +6, 105409 +7, 104899 +0, 116707 +1, 116615 +2, 116705 +3, 116463 +4, 116587 +5, 116705 +6, 116559 +7, 116705 +0, 116707 +1, 116615 +2, 116705 +3, 116463 +4, 116587 +5, 116705 +6, 116559 +7, 116705 +0, 115653 +1, 115969 +2, 115651 +3, 115817 +4, 115533 +5, 116059 +6, 115505 +7, 116059 +0, 16945 +1, 16948 +2, 16948 +3, 16948 +4, 17140 +5, 17332 +6, 115119 +7, 18100 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/run.env index 4d121388cdc2db102c1783c999c4de53c0e99bb8..2e2a2fd96c750ea9322cc0a5bc54f2c95e45a4b6 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/run.env @@ -3,7 +3,7 @@ config=QWEN3_30B family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/train.log index 2be94e7956292673b03e76f94890e64061a3dfe9..f97f4012d00394e661d2502f6718e41d110eda8d 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/train.log @@ -1,18 +1,25 @@ -[23:50:05] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +[01:36:15] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) ============================================================== QWEN3_30B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=768 m=16 L=48 seq=4096 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:50:06.862000 247706 torch/distributed/run.py:766] -W0829 23:50:06.862000 247706 torch/distributed/run.py:766] ***************************************** -W0829 23:50:06.862000 247706 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:50:06.862000 247706 torch/distributed/run.py:766] ***************************************** +W0830 01:36:17.073000 390312 torch/distributed/run.py:766] +W0830 01:36:17.073000 390312 torch/distributed/run.py:766] ***************************************** +W0830 01:36:17.073000 390312 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:36:17.073000 390312 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -561,8 +568,6 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -572,56 +577,51 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to [Gloo] Rank 7 peer ranks. 4Expected number of connected peer ranks is : is connected to 77 - peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 77 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 67 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7[Gloo] Rank peer ranks. 4Expected number of connected peer ranks is : is connected to 77 + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 37 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank5]:[W829 23:50:25.578726332 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W829 23:50:25.578726246 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W829 23:50:25.578734749 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W829 23:50:25.578776076 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W829 23:50:25.578842758 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:50:25.578858460 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:50:25.578897030 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 01:36:34.196130753 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 01:36:34.200690817 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 01:36:34.200726736 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 01:36:34.200755539 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 01:36:34.200783452 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 01:36:34.200822266 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 01:36:34.200871837 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.034 seconds +>>> done with dataset index builder. Compilation time: 0.035 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W829 23:50:25.745088405 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.640 seconds -time to initialize megatron (seconds): 7.199 -[after megatron is initialized] datetime: 2026-08-29 23:50:31 +[rank0]:[W830 01:36:35.364027297 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.769 seconds +time to initialize megatron (seconds): 7.552 +[after megatron is initialized] datetime: 2026-08-30 01:36:41 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -647,139 +647,141 @@ Params for bucket 1 (103022592 elements, 103022592 padded size): Params for bucket 2 (49033984 elements, 49033984 padded size): module.decoder.layers.47.pre_mlp_layernorm.weight module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.45.mlp.router.weight module.decoder.layers.47.self_attention.k_layernorm.weight module.decoder.layers.47.self_attention.q_layernorm.weight module.decoder.layers.47.self_attention.linear_qkv.weight module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.46.self_attention.k_layernorm.weight module.decoder.layers.46.self_attention.q_layernorm.weight - module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.45.pre_mlp_layernorm.weight module.decoder.layers.47.self_attention.linear_proj.weight module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.pre_mlp_layernorm.weight module.decoder.layers.45.self_attention.k_layernorm.weight module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.layers.47.mlp.router.weight + module.decoder.final_layernorm.weight module.decoder.layers.46.mlp.router.weight module.decoder.layers.45.self_attention.q_layernorm.weight - module.decoder.final_layernorm.weight - module.decoder.layers.47.mlp.router.weight Params for bucket 3 (46672384 elements, 46672384 padded size): - module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.45.self_attention.linear_proj.weight - module.decoder.layers.43.mlp.router.weight - module.decoder.layers.43.self_attention.k_layernorm.weight - module.decoder.layers.43.self_attention.linear_proj.weight - module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.44.self_attention.k_layernorm.weight module.decoder.layers.44.pre_mlp_layernorm.weight module.decoder.layers.44.mlp.router.weight - module.decoder.layers.44.self_attention.k_layernorm.weight module.decoder.layers.44.self_attention.q_layernorm.weight module.decoder.layers.44.self_attention.linear_qkv.weight module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.44.self_attention.linear_proj.weight + module.decoder.layers.43.mlp.router.weight + module.decoder.layers.43.self_attention.k_layernorm.weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight module.decoder.layers.43.pre_mlp_layernorm.weight module.decoder.layers.43.self_attention.q_layernorm.weight module.decoder.layers.43.self_attention.linear_qkv.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight Params for bucket 4 (49031936 elements, 49031936 padded size): module.decoder.layers.42.pre_mlp_layernorm.weight module.decoder.layers.41.pre_mlp_layernorm.weight module.decoder.layers.40.mlp.router.weight - module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.k_layernorm.weight module.decoder.layers.42.self_attention.q_layernorm.weight module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.41.self_attention.k_layernorm.weight module.decoder.layers.41.self_attention.q_layernorm.weight module.decoder.layers.41.self_attention.linear_qkv.weight module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.40.pre_mlp_layernorm.weight module.decoder.layers.42.self_attention.linear_proj.weight module.decoder.layers.41.self_attention.linear_proj.weight - module.decoder.layers.41.mlp.router.weight module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.mlp.router.weight module.decoder.layers.40.self_attention.k_layernorm.weight module.decoder.layers.42.mlp.router.weight - module.decoder.layers.40.self_attention.q_layernorm.weight Params for bucket 5 (46672384 elements, 46672384 padded size): - module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.38.mlp.router.weight - module.decoder.layers.38.pre_mlp_layernorm.weight module.decoder.layers.38.self_attention.k_layernorm.weight - module.decoder.layers.38.self_attention.linear_qkv.weight module.decoder.layers.38.self_attention.linear_proj.weight - module.decoder.layers.39.mlp.router.weight - module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.40.self_attention.linear_proj.weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.39.mlp.router.weight module.decoder.layers.39.pre_mlp_layernorm.weight module.decoder.layers.39.self_attention.q_layernorm.weight module.decoder.layers.39.self_attention.linear_qkv.weight module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.39.self_attention.linear_proj.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight module.decoder.layers.38.self_attention.q_layernorm.weight - module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight Params for bucket 6 (49031936 elements, 49031936 padded size): module.decoder.layers.37.pre_mlp_layernorm.weight module.decoder.layers.36.mlp.router.weight - module.decoder.layers.35.self_attention.linear_qkv.weight - module.decoder.layers.35.mlp.router.weight - module.decoder.layers.35.self_attention.k_layernorm.weight module.decoder.layers.36.pre_mlp_layernorm.weight + module.decoder.layers.35.self_attention.k_layernorm.weight module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.35.mlp.router.weight module.decoder.layers.37.self_attention.q_layernorm.weight - module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.q_layernorm.weight module.decoder.layers.36.self_attention.linear_qkv.weight module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.37.self_attention.linear_proj.weight - module.decoder.layers.36.self_attention.q_layernorm.weight module.decoder.layers.36.self_attention.linear_proj.weight module.decoder.layers.35.pre_mlp_layernorm.weight module.decoder.layers.37.mlp.router.weight Params for bucket 7 (46672384 elements, 46672384 padded size): module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.33.mlp.router.weight - module.decoder.layers.33.pre_mlp_layernorm.weight - module.decoder.layers.33.self_attention.k_layernorm.weight - module.decoder.layers.33.self_attention.q_layernorm.weight - module.decoder.layers.33.self_attention.linear_qkv.weight - module.decoder.layers.33.self_attention.linear_proj.weight module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight module.decoder.layers.34.mlp.router.weight module.decoder.layers.35.self_attention.linear_proj.weight - module.decoder.layers.34.self_attention.k_layernorm.weight module.decoder.layers.34.self_attention.q_layernorm.weight module.decoder.layers.34.self_attention.linear_qkv.weight module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.33.mlp.router.weight + module.decoder.layers.33.self_attention.linear_qkv.weight + module.decoder.layers.33.self_attention.k_layernorm.weight module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.33.self_attention.q_layernorm.weight + module.decoder.layers.33.pre_mlp_layernorm.weight Params for bucket 8 (49031936 elements, 49031936 padded size): - module.decoder.layers.32.mlp.router.weight module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.31.mlp.router.weight module.decoder.layers.31.pre_mlp_layernorm.weight module.decoder.layers.30.self_attention.k_layernorm.weight module.decoder.layers.30.self_attention.q_layernorm.weight module.decoder.layers.30.self_attention.linear_qkv.weight - module.decoder.layers.32.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_qkv.weight module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.32.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.31.self_attention.k_layernorm.weight - module.decoder.layers.31.self_attention.q_layernorm.weight - module.decoder.layers.30.mlp.router.weight - module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.mlp.router.weight module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.self_attention.q_layernorm.weight module.decoder.layers.31.self_attention.linear_proj.weight - module.decoder.layers.31.mlp.router.weight + module.decoder.layers.32.mlp.router.weight module.decoder.layers.30.pre_mlp_layernorm.weight Params for bucket 9 (46672384 elements, 46672384 padded size): module.decoder.layers.30.self_attention.linear_proj.weight - module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.mlp.router.weight + module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.weight + module.decoder.layers.28.self_attention.linear_proj.weight module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.29.mlp.router.weight module.decoder.layers.29.self_attention.k_layernorm.weight @@ -788,63 +790,61 @@ Params for bucket 9 (46672384 elements, 46672384 padded size): module.decoder.layers.29.self_attention.linear_qkv.weight module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.29.self_attention.linear_proj.weight - module.decoder.layers.28.mlp.router.weight module.decoder.layers.28.pre_mlp_layernorm.weight - module.decoder.layers.28.self_attention.linear_qkv.weight - module.decoder.layers.28.self_attention.linear_proj.weight - module.decoder.layers.28.self_attention.q_layernorm.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight Params for bucket 10 (49031936 elements, 49031936 padded size): module.decoder.layers.27.mlp.router.weight - module.decoder.layers.26.pre_mlp_layernorm.weight module.decoder.layers.27.pre_mlp_layernorm.weight - module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.pre_mlp_layernorm.weight module.decoder.layers.25.pre_mlp_layernorm.weight module.decoder.layers.25.self_attention.k_layernorm.weight - module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.q_layernorm.weight module.decoder.layers.27.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.k_layernorm.weight module.decoder.layers.26.self_attention.q_layernorm.weight + module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.26.self_attention.linear_qkv.weight module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.25.self_attention.linear_qkv.weight - module.decoder.layers.27.self_attention.q_layernorm.weight module.decoder.layers.27.self_attention.linear_proj.weight module.decoder.layers.26.self_attention.linear_proj.weight - module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.26.mlp.router.weight module.decoder.layers.25.mlp.router.weight Params for bucket 11 (46672384 elements, 46672384 padded size): + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.23.mlp.router.weight module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.self_attention.k_layernorm.weight module.decoder.layers.23.self_attention.q_layernorm.weight module.decoder.layers.23.self_attention.linear_qkv.weight module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.23.self_attention.linear_proj.weight - module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.24.mlp.router.weight - module.decoder.layers.25.self_attention.linear_proj.weight module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.24.mlp.router.weight module.decoder.layers.24.self_attention.k_layernorm.weight module.decoder.layers.24.self_attention.q_layernorm.weight module.decoder.layers.24.self_attention.linear_qkv.weight module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.24.self_attention.linear_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.21.pre_mlp_layernorm.weight module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.21.self_attention.k_layernorm.weight - module.decoder.layers.21.self_attention.linear_qkv.weight module.decoder.layers.20.self_attention.k_layernorm.weight - module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.linear_qkv.weight module.decoder.layers.22.self_attention.k_layernorm.weight + module.decoder.layers.22.self_attention.linear_qkv.weight module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.k_layernorm.weight module.decoder.layers.21.self_attention.q_layernorm.weight - module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_qkv.weight module.decoder.layers.20.self_attention.q_layernorm.weight - module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.22.self_attention.q_layernorm.weight module.decoder.layers.21.self_attention.linear_proj.weight module.decoder.layers.22.self_attention.linear_proj.weight @@ -852,158 +852,165 @@ Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.21.mlp.router.weight Params for bucket 13 (46672384 elements, 46672384 padded size): module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.k_layernorm.weight module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.self_attention.k_layernorm.weight - module.decoder.layers.19.self_attention.q_layernorm.weight module.decoder.layers.19.self_attention.linear_qkv.weight module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.20.self_attention.linear_proj.weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.18.mlp.router.weight module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.self_attention.k_layernorm.weight module.decoder.layers.18.self_attention.q_layernorm.weight module.decoder.layers.18.self_attention.linear_qkv.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.q_layernorm.weight module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.18.self_attention.linear_proj.weight Params for bucket 14 (49031936 elements, 49031936 padded size): module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.15.pre_mlp_layernorm.weight module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight module.decoder.layers.15.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.weight - module.decoder.layers.17.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.q_layernorm.weight module.decoder.layers.15.self_attention.linear_qkv.weight - module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.16.self_attention.k_layernorm.weight module.decoder.layers.16.self_attention.q_layernorm.weight module.decoder.layers.16.self_attention.linear_qkv.weight module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.linear_proj.weight module.decoder.layers.17.self_attention.linear_proj.weight module.decoder.layers.15.mlp.router.weight module.decoder.layers.16.mlp.router.weight Params for bucket 15 (46672384 elements, 46672384 padded size): module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.self_attention.k_layernorm.weight - module.decoder.layers.14.mlp.router.weight module.decoder.layers.14.self_attention.q_layernorm.weight module.decoder.layers.14.self_attention.linear_qkv.weight - module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.k_layernorm.weight module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.pre_mlp_layernorm.weight module.decoder.layers.13.self_attention.k_layernorm.weight module.decoder.layers.13.self_attention.q_layernorm.weight module.decoder.layers.13.self_attention.linear_qkv.weight - module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight Params for bucket 16 (49031936 elements, 49031936 padded size): module.decoder.layers.12.mlp.router.weight - module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.12.self_attention.k_layernorm.weight - module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.10.self_attention.q_layernorm.weight - module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.k_layernorm.weight module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.q_layernorm.weight module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.10.mlp.router.weight - module.decoder.layers.11.mlp.router.weight Params for bucket 17 (46672384 elements, 46672384 padded size): module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight Params for bucket 18 (49031936 elements, 49031936 padded size): module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.q_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.router.weight module.decoder.layers.6.mlp.router.weight Params for bucket 19 (46672384 elements, 46672384 padded size): module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.q_layernorm.weight - module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight Params for bucket 20 (49031936 elements, 49031936 padded size): module.decoder.layers.2.mlp.router.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.k_layernorm.weight module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.k_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 21 (111413248 elements, 111413248 padded size): module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight module.embedding.word_embeddings.weight module.decoder.layers.0.self_attention.linear_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 Params for bucket 1 (40894464 elements, 40894464 padded size): + module.decoder.layers.47.mlp.experts.linear_fc2.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight10 + module.decoder.layers.47.mlp.experts.linear_fc2.weight7 + module.decoder.layers.47.mlp.experts.linear_fc2.weight15 + module.decoder.layers.47.mlp.experts.linear_fc1.weight2 + module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc2.weight12 module.decoder.layers.47.mlp.experts.linear_fc2.weight1 module.decoder.layers.47.mlp.experts.linear_fc2.weight9 module.decoder.layers.47.mlp.experts.linear_fc1.weight4 @@ -1018,30 +1025,30 @@ Params for bucket 1 (40894464 elements, 40894464 padded size): module.decoder.layers.47.mlp.experts.linear_fc2.weight5 module.decoder.layers.47.mlp.experts.linear_fc2.weight13 module.decoder.layers.47.mlp.experts.linear_fc1.weight0 - module.decoder.layers.47.mlp.experts.linear_fc2.weight2 - module.decoder.layers.47.mlp.experts.linear_fc2.weight10 - module.decoder.layers.47.mlp.experts.linear_fc2.weight7 - module.decoder.layers.47.mlp.experts.linear_fc2.weight15 - module.decoder.layers.47.mlp.experts.linear_fc1.weight2 - module.decoder.layers.47.mlp.experts.linear_fc2.weight4 - module.decoder.layers.47.mlp.experts.linear_fc2.weight12 Params for bucket 2 (40894464 elements, 40894464 padded size): - module.decoder.layers.47.mlp.experts.linear_fc1.weight6 - module.decoder.layers.47.mlp.experts.linear_fc1.weight8 - module.decoder.layers.47.mlp.experts.linear_fc1.weight10 - module.decoder.layers.47.mlp.experts.linear_fc1.weight12 - module.decoder.layers.47.mlp.experts.linear_fc1.weight14 - module.decoder.layers.46.mlp.experts.linear_fc2.weight0 - module.decoder.layers.46.mlp.experts.linear_fc2.weight2 module.decoder.layers.47.mlp.experts.linear_fc1.weight5 - module.decoder.layers.47.mlp.experts.linear_fc1.weight9 module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 module.decoder.layers.47.mlp.experts.linear_fc1.weight11 module.decoder.layers.47.mlp.experts.linear_fc1.weight13 module.decoder.layers.47.mlp.experts.linear_fc1.weight15 module.decoder.layers.46.mlp.experts.linear_fc2.weight1 module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 Params for bucket 3 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc2.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight6 + module.decoder.layers.46.mlp.experts.linear_fc2.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight3 + module.decoder.layers.46.mlp.experts.linear_fc2.weight5 + module.decoder.layers.46.mlp.experts.linear_fc2.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight0 module.decoder.layers.46.mlp.experts.linear_fc2.weight10 module.decoder.layers.46.mlp.experts.linear_fc1.weight5 module.decoder.layers.46.mlp.experts.linear_fc2.weight7 @@ -1054,23 +1061,7 @@ Params for bucket 3 (40894464 elements, 40894464 padded size): module.decoder.layers.46.mlp.experts.linear_fc2.weight6 module.decoder.layers.46.mlp.experts.linear_fc2.weight14 module.decoder.layers.46.mlp.experts.linear_fc1.weight1 - module.decoder.layers.46.mlp.experts.linear_fc2.weight11 - module.decoder.layers.46.mlp.experts.linear_fc1.weight6 - module.decoder.layers.46.mlp.experts.linear_fc2.weight8 - module.decoder.layers.46.mlp.experts.linear_fc1.weight3 - module.decoder.layers.46.mlp.experts.linear_fc2.weight5 - module.decoder.layers.46.mlp.experts.linear_fc2.weight13 - module.decoder.layers.46.mlp.experts.linear_fc1.weight0 Params for bucket 4 (40894464 elements, 40894464 padded size): - module.decoder.layers.46.mlp.experts.linear_fc1.weight7 - module.decoder.layers.46.mlp.experts.linear_fc1.weight11 - module.decoder.layers.46.mlp.experts.linear_fc1.weight9 - module.decoder.layers.46.mlp.experts.linear_fc1.weight13 - module.decoder.layers.46.mlp.experts.linear_fc1.weight15 - module.decoder.layers.45.mlp.experts.linear_fc2.weight1 - module.decoder.layers.45.mlp.experts.linear_fc2.weight3 - module.decoder.layers.45.mlp.experts.linear_fc2.weight5 - module.decoder.layers.45.mlp.experts.linear_fc2.weight7 module.decoder.layers.46.mlp.experts.linear_fc1.weight10 module.decoder.layers.46.mlp.experts.linear_fc1.weight8 module.decoder.layers.46.mlp.experts.linear_fc1.weight12 @@ -1079,7 +1070,24 @@ Params for bucket 4 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc2.weight2 module.decoder.layers.45.mlp.experts.linear_fc2.weight4 module.decoder.layers.45.mlp.experts.linear_fc2.weight6 + module.decoder.layers.46.mlp.experts.linear_fc1.weight7 + module.decoder.layers.46.mlp.experts.linear_fc1.weight9 + module.decoder.layers.46.mlp.experts.linear_fc1.weight11 + module.decoder.layers.46.mlp.experts.linear_fc1.weight13 + module.decoder.layers.46.mlp.experts.linear_fc1.weight15 + module.decoder.layers.45.mlp.experts.linear_fc2.weight1 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 Params for bucket 5 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc2.weight8 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 + module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight1 + module.decoder.layers.45.mlp.experts.linear_fc1.weight3 + module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight7 module.decoder.layers.45.mlp.experts.linear_fc2.weight11 module.decoder.layers.45.mlp.experts.linear_fc2.weight9 module.decoder.layers.45.mlp.experts.linear_fc2.weight13 @@ -1089,15 +1097,15 @@ Params for bucket 5 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc1.weight4 module.decoder.layers.45.mlp.experts.linear_fc1.weight6 module.decoder.layers.45.mlp.experts.linear_fc1.weight8 - module.decoder.layers.45.mlp.experts.linear_fc2.weight8 - module.decoder.layers.45.mlp.experts.linear_fc2.weight10 - module.decoder.layers.45.mlp.experts.linear_fc2.weight12 - module.decoder.layers.45.mlp.experts.linear_fc2.weight14 - module.decoder.layers.45.mlp.experts.linear_fc1.weight1 - module.decoder.layers.45.mlp.experts.linear_fc1.weight3 - module.decoder.layers.45.mlp.experts.linear_fc1.weight5 - module.decoder.layers.45.mlp.experts.linear_fc1.weight7 Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight6 + module.decoder.layers.45.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc2.weight3 + module.decoder.layers.44.mlp.experts.linear_fc2.weight11 module.decoder.layers.45.mlp.experts.linear_fc1.weight14 module.decoder.layers.44.mlp.experts.linear_fc2.weight0 module.decoder.layers.44.mlp.experts.linear_fc2.weight8 @@ -1109,31 +1117,30 @@ Params for bucket 6 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc2.weight7 module.decoder.layers.45.mlp.experts.linear_fc1.weight10 module.decoder.layers.44.mlp.experts.linear_fc2.weight4 - module.decoder.layers.45.mlp.experts.linear_fc1.weight15 - module.decoder.layers.44.mlp.experts.linear_fc2.weight1 - module.decoder.layers.44.mlp.experts.linear_fc2.weight9 - module.decoder.layers.45.mlp.experts.linear_fc1.weight12 - module.decoder.layers.44.mlp.experts.linear_fc2.weight6 - module.decoder.layers.45.mlp.experts.linear_fc1.weight9 - module.decoder.layers.44.mlp.experts.linear_fc2.weight3 - module.decoder.layers.44.mlp.experts.linear_fc2.weight11 Params for bucket 7 (40894464 elements, 40894464 padded size): - module.decoder.layers.44.mlp.experts.linear_fc2.weight12 - module.decoder.layers.44.mlp.experts.linear_fc2.weight14 - module.decoder.layers.44.mlp.experts.linear_fc1.weight1 - module.decoder.layers.44.mlp.experts.linear_fc1.weight3 - module.decoder.layers.44.mlp.experts.linear_fc1.weight5 - module.decoder.layers.44.mlp.experts.linear_fc1.weight7 - module.decoder.layers.44.mlp.experts.linear_fc1.weight9 - module.decoder.layers.44.mlp.experts.linear_fc2.weight13 module.decoder.layers.44.mlp.experts.linear_fc2.weight15 module.decoder.layers.44.mlp.experts.linear_fc1.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight13 module.decoder.layers.44.mlp.experts.linear_fc1.weight2 module.decoder.layers.44.mlp.experts.linear_fc1.weight4 module.decoder.layers.44.mlp.experts.linear_fc1.weight6 module.decoder.layers.44.mlp.experts.linear_fc1.weight8 module.decoder.layers.44.mlp.experts.linear_fc1.weight10 + module.decoder.layers.44.mlp.experts.linear_fc2.weight12 + module.decoder.layers.44.mlp.experts.linear_fc2.weight14 + module.decoder.layers.44.mlp.experts.linear_fc1.weight1 + module.decoder.layers.44.mlp.experts.linear_fc1.weight3 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.43.mlp.experts.linear_fc2.weight2 + module.decoder.layers.43.mlp.experts.linear_fc2.weight10 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc2.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight12 module.decoder.layers.44.mlp.experts.linear_fc1.weight15 module.decoder.layers.43.mlp.experts.linear_fc2.weight1 module.decoder.layers.43.mlp.experts.linear_fc2.weight9 @@ -1148,69 +1155,57 @@ Params for bucket 8 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc1.weight11 module.decoder.layers.43.mlp.experts.linear_fc2.weight5 module.decoder.layers.43.mlp.experts.linear_fc2.weight13 - module.decoder.layers.43.mlp.experts.linear_fc2.weight2 - module.decoder.layers.43.mlp.experts.linear_fc2.weight10 - module.decoder.layers.44.mlp.experts.linear_fc1.weight13 - module.decoder.layers.43.mlp.experts.linear_fc2.weight7 - module.decoder.layers.43.mlp.experts.linear_fc2.weight15 - module.decoder.layers.43.mlp.experts.linear_fc2.weight4 - module.decoder.layers.43.mlp.experts.linear_fc2.weight12 Params for bucket 9 (40894464 elements, 40894464 padded size): - module.decoder.layers.43.mlp.experts.linear_fc1.weight0 - module.decoder.layers.43.mlp.experts.linear_fc1.weight2 - module.decoder.layers.43.mlp.experts.linear_fc1.weight4 - module.decoder.layers.43.mlp.experts.linear_fc1.weight6 - module.decoder.layers.43.mlp.experts.linear_fc1.weight8 - module.decoder.layers.43.mlp.experts.linear_fc1.weight10 - module.decoder.layers.43.mlp.experts.linear_fc1.weight12 module.decoder.layers.43.mlp.experts.linear_fc1.weight3 module.decoder.layers.43.mlp.experts.linear_fc1.weight1 module.decoder.layers.43.mlp.experts.linear_fc1.weight5 module.decoder.layers.43.mlp.experts.linear_fc1.weight7 module.decoder.layers.43.mlp.experts.linear_fc1.weight9 module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 + module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 + module.decoder.layers.43.mlp.experts.linear_fc1.weight6 + module.decoder.layers.43.mlp.experts.linear_fc1.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight10 + module.decoder.layers.43.mlp.experts.linear_fc1.weight12 Params for bucket 10 (40894464 elements, 40894464 padded size): - module.decoder.layers.42.mlp.experts.linear_fc2.weight1 - module.decoder.layers.42.mlp.experts.linear_fc2.weight9 - module.decoder.layers.42.mlp.experts.linear_fc2.weight6 - module.decoder.layers.42.mlp.experts.linear_fc2.weight14 - module.decoder.layers.42.mlp.experts.linear_fc1.weight1 - module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 module.decoder.layers.42.mlp.experts.linear_fc2.weight3 module.decoder.layers.42.mlp.experts.linear_fc2.weight11 module.decoder.layers.42.mlp.experts.linear_fc2.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight8 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 module.decoder.layers.42.mlp.experts.linear_fc2.weight5 module.decoder.layers.42.mlp.experts.linear_fc2.weight13 module.decoder.layers.42.mlp.experts.linear_fc1.weight0 - module.decoder.layers.43.mlp.experts.linear_fc1.weight13 module.decoder.layers.42.mlp.experts.linear_fc2.weight2 module.decoder.layers.42.mlp.experts.linear_fc2.weight10 module.decoder.layers.42.mlp.experts.linear_fc2.weight7 module.decoder.layers.42.mlp.experts.linear_fc2.weight15 - module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 module.decoder.layers.42.mlp.experts.linear_fc2.weight4 module.decoder.layers.42.mlp.experts.linear_fc2.weight12 + module.decoder.layers.42.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc2.weight9 + module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.42.mlp.experts.linear_fc1.weight1 Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight2 - module.decoder.layers.42.mlp.experts.linear_fc1.weight4 module.decoder.layers.42.mlp.experts.linear_fc1.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight4 module.decoder.layers.42.mlp.experts.linear_fc1.weight8 module.decoder.layers.42.mlp.experts.linear_fc1.weight10 module.decoder.layers.42.mlp.experts.linear_fc1.weight12 module.decoder.layers.42.mlp.experts.linear_fc1.weight14 - module.decoder.layers.42.mlp.experts.linear_fc1.weight5 module.decoder.layers.42.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight5 module.decoder.layers.42.mlp.experts.linear_fc1.weight7 module.decoder.layers.42.mlp.experts.linear_fc1.weight9 module.decoder.layers.42.mlp.experts.linear_fc1.weight11 module.decoder.layers.42.mlp.experts.linear_fc1.weight13 Params for bucket 12 (40894464 elements, 40894464 padded size): - module.decoder.layers.41.mlp.experts.linear_fc2.weight2 - module.decoder.layers.41.mlp.experts.linear_fc2.weight10 - module.decoder.layers.41.mlp.experts.linear_fc2.weight7 - module.decoder.layers.41.mlp.experts.linear_fc2.weight15 - module.decoder.layers.41.mlp.experts.linear_fc1.weight2 module.decoder.layers.41.mlp.experts.linear_fc2.weight4 module.decoder.layers.41.mlp.experts.linear_fc2.weight12 module.decoder.layers.41.mlp.experts.linear_fc2.weight1 @@ -1227,6 +1222,11 @@ Params for bucket 12 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc2.weight5 module.decoder.layers.41.mlp.experts.linear_fc2.weight13 module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight2 + module.decoder.layers.41.mlp.experts.linear_fc2.weight10 + module.decoder.layers.41.mlp.experts.linear_fc2.weight7 + module.decoder.layers.41.mlp.experts.linear_fc2.weight15 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 Params for bucket 13 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight7 module.decoder.layers.41.mlp.experts.linear_fc1.weight5 @@ -1243,10 +1243,6 @@ Params for bucket 13 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight14 module.decoder.layers.40.mlp.experts.linear_fc2.weight0 Params for bucket 14 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc2.weight3 - module.decoder.layers.40.mlp.experts.linear_fc2.weight11 - module.decoder.layers.40.mlp.experts.linear_fc2.weight8 - module.decoder.layers.40.mlp.experts.linear_fc1.weight3 module.decoder.layers.40.mlp.experts.linear_fc2.weight5 module.decoder.layers.40.mlp.experts.linear_fc2.weight13 module.decoder.layers.40.mlp.experts.linear_fc1.weight0 @@ -1263,17 +1259,21 @@ Params for bucket 14 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc2.weight6 module.decoder.layers.40.mlp.experts.linear_fc2.weight14 module.decoder.layers.40.mlp.experts.linear_fc1.weight1 + module.decoder.layers.40.mlp.experts.linear_fc2.weight3 + module.decoder.layers.40.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc2.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 Params for bucket 15 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc1.weight6 - module.decoder.layers.40.mlp.experts.linear_fc1.weight10 module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 module.decoder.layers.40.mlp.experts.linear_fc1.weight12 module.decoder.layers.40.mlp.experts.linear_fc1.weight14 module.decoder.layers.39.mlp.experts.linear_fc2.weight0 module.decoder.layers.39.mlp.experts.linear_fc2.weight2 module.decoder.layers.39.mlp.experts.linear_fc2.weight4 - module.decoder.layers.40.mlp.experts.linear_fc1.weight7 module.decoder.layers.40.mlp.experts.linear_fc1.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight7 module.decoder.layers.40.mlp.experts.linear_fc1.weight11 module.decoder.layers.40.mlp.experts.linear_fc1.weight13 module.decoder.layers.40.mlp.experts.linear_fc1.weight15 @@ -1282,16 +1282,16 @@ Params for bucket 15 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc2.weight5 Params for bucket 16 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc2.weight6 - module.decoder.layers.39.mlp.experts.linear_fc2.weight8 module.decoder.layers.39.mlp.experts.linear_fc2.weight10 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 module.decoder.layers.39.mlp.experts.linear_fc2.weight12 module.decoder.layers.39.mlp.experts.linear_fc2.weight14 module.decoder.layers.39.mlp.experts.linear_fc1.weight1 module.decoder.layers.39.mlp.experts.linear_fc1.weight3 module.decoder.layers.39.mlp.experts.linear_fc1.weight5 module.decoder.layers.39.mlp.experts.linear_fc1.weight7 - module.decoder.layers.39.mlp.experts.linear_fc2.weight9 module.decoder.layers.39.mlp.experts.linear_fc2.weight7 + module.decoder.layers.39.mlp.experts.linear_fc2.weight9 module.decoder.layers.39.mlp.experts.linear_fc2.weight11 module.decoder.layers.39.mlp.experts.linear_fc2.weight13 module.decoder.layers.39.mlp.experts.linear_fc2.weight15 @@ -1300,8 +1300,8 @@ Params for bucket 16 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc1.weight4 module.decoder.layers.39.mlp.experts.linear_fc1.weight6 Params for bucket 17 (40894464 elements, 40894464 padded size): - module.decoder.layers.39.mlp.experts.linear_fc1.weight9 module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 module.decoder.layers.39.mlp.experts.linear_fc1.weight13 module.decoder.layers.39.mlp.experts.linear_fc1.weight15 module.decoder.layers.38.mlp.experts.linear_fc2.weight1 @@ -1319,23 +1319,32 @@ Params for bucket 17 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc2.weight6 module.decoder.layers.38.mlp.experts.linear_fc2.weight8 Params for bucket 18 (40894464 elements, 40894464 padded size): - module.decoder.layers.38.mlp.experts.linear_fc2.weight11 module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 module.decoder.layers.38.mlp.experts.linear_fc2.weight15 - module.decoder.layers.38.mlp.experts.linear_fc1.weight0 - module.decoder.layers.38.mlp.experts.linear_fc1.weight2 - module.decoder.layers.38.mlp.experts.linear_fc1.weight4 - module.decoder.layers.38.mlp.experts.linear_fc1.weight6 - module.decoder.layers.38.mlp.experts.linear_fc1.weight8 - module.decoder.layers.38.mlp.experts.linear_fc2.weight10 - module.decoder.layers.38.mlp.experts.linear_fc2.weight14 - module.decoder.layers.38.mlp.experts.linear_fc2.weight12 module.decoder.layers.38.mlp.experts.linear_fc1.weight1 module.decoder.layers.38.mlp.experts.linear_fc1.weight3 module.decoder.layers.38.mlp.experts.linear_fc1.weight5 module.decoder.layers.38.mlp.experts.linear_fc1.weight7 module.decoder.layers.38.mlp.experts.linear_fc1.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight10 + module.decoder.layers.38.mlp.experts.linear_fc2.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 Params for bucket 19 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 + module.decoder.layers.37.mlp.experts.linear_fc2.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.38.mlp.experts.linear_fc1.weight15 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 module.decoder.layers.38.mlp.experts.linear_fc1.weight12 module.decoder.layers.37.mlp.experts.linear_fc2.weight2 module.decoder.layers.37.mlp.experts.linear_fc2.weight10 @@ -1347,16 +1356,14 @@ Params for bucket 19 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc2.weight1 module.decoder.layers.37.mlp.experts.linear_fc2.weight9 module.decoder.layers.37.mlp.experts.linear_fc2.weight6 - module.decoder.layers.38.mlp.experts.linear_fc1.weight13 - module.decoder.layers.37.mlp.experts.linear_fc2.weight3 - module.decoder.layers.37.mlp.experts.linear_fc2.weight11 - module.decoder.layers.38.mlp.experts.linear_fc1.weight10 - module.decoder.layers.37.mlp.experts.linear_fc2.weight0 - module.decoder.layers.37.mlp.experts.linear_fc2.weight8 - module.decoder.layers.38.mlp.experts.linear_fc1.weight15 - module.decoder.layers.37.mlp.experts.linear_fc2.weight5 - module.decoder.layers.37.mlp.experts.linear_fc2.weight13 Params for bucket 20 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc1.weight4 + module.decoder.layers.37.mlp.experts.linear_fc1.weight6 + module.decoder.layers.37.mlp.experts.linear_fc1.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight10 module.decoder.layers.37.mlp.experts.linear_fc2.weight14 module.decoder.layers.37.mlp.experts.linear_fc1.weight1 module.decoder.layers.37.mlp.experts.linear_fc1.weight3 @@ -1364,14 +1371,14 @@ Params for bucket 20 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc1.weight7 module.decoder.layers.37.mlp.experts.linear_fc1.weight9 module.decoder.layers.37.mlp.experts.linear_fc1.weight11 - module.decoder.layers.37.mlp.experts.linear_fc1.weight0 - module.decoder.layers.37.mlp.experts.linear_fc1.weight2 - module.decoder.layers.37.mlp.experts.linear_fc2.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight4 - module.decoder.layers.37.mlp.experts.linear_fc1.weight6 - module.decoder.layers.37.mlp.experts.linear_fc1.weight8 - module.decoder.layers.37.mlp.experts.linear_fc1.weight10 Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.37.mlp.experts.linear_fc1.weight14 + module.decoder.layers.36.mlp.experts.linear_fc2.weight4 + module.decoder.layers.36.mlp.experts.linear_fc2.weight12 + module.decoder.layers.36.mlp.experts.linear_fc2.weight1 + module.decoder.layers.36.mlp.experts.linear_fc2.weight9 + module.decoder.layers.36.mlp.experts.linear_fc2.weight6 + module.decoder.layers.36.mlp.experts.linear_fc2.weight14 module.decoder.layers.37.mlp.experts.linear_fc1.weight13 module.decoder.layers.36.mlp.experts.linear_fc2.weight3 module.decoder.layers.36.mlp.experts.linear_fc2.weight11 @@ -1386,28 +1393,29 @@ Params for bucket 21 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc2.weight10 module.decoder.layers.36.mlp.experts.linear_fc2.weight7 module.decoder.layers.36.mlp.experts.linear_fc2.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight14 - module.decoder.layers.36.mlp.experts.linear_fc2.weight4 - module.decoder.layers.36.mlp.experts.linear_fc2.weight12 - module.decoder.layers.36.mlp.experts.linear_fc2.weight1 - module.decoder.layers.36.mlp.experts.linear_fc2.weight9 - module.decoder.layers.36.mlp.experts.linear_fc2.weight6 - module.decoder.layers.36.mlp.experts.linear_fc2.weight14 Params for bucket 22 (40894464 elements, 40894464 padded size): - module.decoder.layers.36.mlp.experts.linear_fc1.weight4 - module.decoder.layers.36.mlp.experts.linear_fc1.weight2 - module.decoder.layers.36.mlp.experts.linear_fc1.weight6 - module.decoder.layers.36.mlp.experts.linear_fc1.weight8 - module.decoder.layers.36.mlp.experts.linear_fc1.weight10 - module.decoder.layers.36.mlp.experts.linear_fc1.weight12 module.decoder.layers.36.mlp.experts.linear_fc1.weight1 - module.decoder.layers.36.mlp.experts.linear_fc1.weight3 module.decoder.layers.36.mlp.experts.linear_fc1.weight5 + module.decoder.layers.36.mlp.experts.linear_fc1.weight3 module.decoder.layers.36.mlp.experts.linear_fc1.weight7 module.decoder.layers.36.mlp.experts.linear_fc1.weight9 module.decoder.layers.36.mlp.experts.linear_fc1.weight11 module.decoder.layers.36.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight4 + module.decoder.layers.36.mlp.experts.linear_fc1.weight2 + module.decoder.layers.36.mlp.experts.linear_fc1.weight6 + module.decoder.layers.36.mlp.experts.linear_fc1.weight8 + module.decoder.layers.36.mlp.experts.linear_fc1.weight10 + module.decoder.layers.36.mlp.experts.linear_fc1.weight12 Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 module.decoder.layers.36.mlp.experts.linear_fc1.weight14 module.decoder.layers.35.mlp.experts.linear_fc2.weight3 module.decoder.layers.35.mlp.experts.linear_fc2.weight11 @@ -1421,21 +1429,7 @@ Params for bucket 23 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc2.weight7 module.decoder.layers.35.mlp.experts.linear_fc2.weight15 module.decoder.layers.35.mlp.experts.linear_fc1.weight2 - module.decoder.layers.36.mlp.experts.linear_fc1.weight15 - module.decoder.layers.35.mlp.experts.linear_fc2.weight4 - module.decoder.layers.35.mlp.experts.linear_fc2.weight12 - module.decoder.layers.35.mlp.experts.linear_fc2.weight1 - module.decoder.layers.35.mlp.experts.linear_fc2.weight9 - module.decoder.layers.35.mlp.experts.linear_fc2.weight6 - module.decoder.layers.35.mlp.experts.linear_fc2.weight14 - module.decoder.layers.35.mlp.experts.linear_fc1.weight1 Params for bucket 24 (40894464 elements, 40894464 padded size): - module.decoder.layers.35.mlp.experts.linear_fc1.weight4 - module.decoder.layers.35.mlp.experts.linear_fc1.weight6 - module.decoder.layers.35.mlp.experts.linear_fc1.weight8 - module.decoder.layers.35.mlp.experts.linear_fc1.weight10 - module.decoder.layers.35.mlp.experts.linear_fc1.weight12 - module.decoder.layers.35.mlp.experts.linear_fc1.weight14 module.decoder.layers.35.mlp.experts.linear_fc1.weight3 module.decoder.layers.35.mlp.experts.linear_fc1.weight5 module.decoder.layers.35.mlp.experts.linear_fc1.weight7 @@ -1443,7 +1437,21 @@ Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight11 module.decoder.layers.35.mlp.experts.linear_fc1.weight13 module.decoder.layers.35.mlp.experts.linear_fc1.weight15 + module.decoder.layers.35.mlp.experts.linear_fc1.weight6 + module.decoder.layers.35.mlp.experts.linear_fc1.weight4 + module.decoder.layers.35.mlp.experts.linear_fc1.weight8 + module.decoder.layers.35.mlp.experts.linear_fc1.weight10 + module.decoder.layers.35.mlp.experts.linear_fc1.weight12 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight6 + module.decoder.layers.34.mlp.experts.linear_fc2.weight14 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight2 module.decoder.layers.34.mlp.experts.linear_fc2.weight0 module.decoder.layers.34.mlp.experts.linear_fc2.weight8 module.decoder.layers.34.mlp.experts.linear_fc2.weight5 @@ -1457,23 +1465,7 @@ Params for bucket 25 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc2.weight4 module.decoder.layers.34.mlp.experts.linear_fc2.weight12 module.decoder.layers.34.mlp.experts.linear_fc1.weight3 - module.decoder.layers.34.mlp.experts.linear_fc2.weight1 - module.decoder.layers.34.mlp.experts.linear_fc2.weight9 - module.decoder.layers.34.mlp.experts.linear_fc1.weight0 - module.decoder.layers.34.mlp.experts.linear_fc2.weight6 - module.decoder.layers.34.mlp.experts.linear_fc2.weight14 - module.decoder.layers.34.mlp.experts.linear_fc2.weight3 - module.decoder.layers.34.mlp.experts.linear_fc2.weight11 - module.decoder.layers.34.mlp.experts.linear_fc1.weight2 Params for bucket 26 (40894464 elements, 40894464 padded size): - module.decoder.layers.34.mlp.experts.linear_fc1.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight7 - module.decoder.layers.34.mlp.experts.linear_fc1.weight9 - module.decoder.layers.34.mlp.experts.linear_fc1.weight11 - module.decoder.layers.34.mlp.experts.linear_fc1.weight13 - module.decoder.layers.34.mlp.experts.linear_fc1.weight15 - module.decoder.layers.33.mlp.experts.linear_fc2.weight1 - module.decoder.layers.33.mlp.experts.linear_fc2.weight3 module.decoder.layers.34.mlp.experts.linear_fc1.weight8 module.decoder.layers.34.mlp.experts.linear_fc1.weight6 module.decoder.layers.34.mlp.experts.linear_fc1.weight10 @@ -1481,25 +1473,33 @@ Params for bucket 26 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc1.weight14 module.decoder.layers.33.mlp.experts.linear_fc2.weight0 module.decoder.layers.33.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc1.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight11 + module.decoder.layers.34.mlp.experts.linear_fc1.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight1 + module.decoder.layers.33.mlp.experts.linear_fc2.weight3 Params for bucket 27 (40894464 elements, 40894464 padded size): - module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight10 module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 module.decoder.layers.33.mlp.experts.linear_fc2.weight6 module.decoder.layers.33.mlp.experts.linear_fc2.weight14 - module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc1.weight4 module.decoder.layers.33.mlp.experts.linear_fc2.weight11 - module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 module.decoder.layers.33.mlp.experts.linear_fc2.weight8 + module.decoder.layers.33.mlp.experts.linear_fc1.weight6 module.decoder.layers.33.mlp.experts.linear_fc2.weight5 module.decoder.layers.33.mlp.experts.linear_fc2.weight13 - module.decoder.layers.33.mlp.experts.linear_fc1.weight4 - module.decoder.layers.33.mlp.experts.linear_fc2.weight10 - module.decoder.layers.33.mlp.experts.linear_fc1.weight1 - module.decoder.layers.33.mlp.experts.linear_fc2.weight7 - module.decoder.layers.33.mlp.experts.linear_fc2.weight15 - module.decoder.layers.33.mlp.experts.linear_fc1.weight6 - module.decoder.layers.33.mlp.experts.linear_fc2.weight4 - module.decoder.layers.33.mlp.experts.linear_fc2.weight12 module.decoder.layers.33.mlp.experts.linear_fc1.weight3 Params for bucket 28 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc1.weight10 @@ -1539,58 +1539,58 @@ Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight8 Params for bucket 30 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight11 + module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.31.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight12 module.decoder.layers.31.mlp.experts.linear_fc2.weight2 module.decoder.layers.31.mlp.experts.linear_fc2.weight10 - module.decoder.layers.32.mlp.experts.linear_fc1.weight10 + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.31.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc1.weight15 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 module.decoder.layers.31.mlp.experts.linear_fc2.weight4 - module.decoder.layers.32.mlp.experts.linear_fc1.weight12 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 module.decoder.layers.31.mlp.experts.linear_fc2.weight1 module.decoder.layers.31.mlp.experts.linear_fc2.weight9 - module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.31.mlp.experts.linear_fc2.weight6 - module.decoder.layers.32.mlp.experts.linear_fc1.weight14 - module.decoder.layers.31.mlp.experts.linear_fc2.weight3 - module.decoder.layers.31.mlp.experts.linear_fc2.weight11 - module.decoder.layers.32.mlp.experts.linear_fc1.weight11 - module.decoder.layers.31.mlp.experts.linear_fc2.weight0 - module.decoder.layers.31.mlp.experts.linear_fc2.weight8 - module.decoder.layers.31.mlp.experts.linear_fc2.weight5 Params for bucket 31 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc2.weight12 - module.decoder.layers.31.mlp.experts.linear_fc2.weight14 - module.decoder.layers.31.mlp.experts.linear_fc1.weight1 - module.decoder.layers.31.mlp.experts.linear_fc1.weight3 - module.decoder.layers.31.mlp.experts.linear_fc1.weight5 - module.decoder.layers.31.mlp.experts.linear_fc1.weight7 - module.decoder.layers.31.mlp.experts.linear_fc1.weight9 - module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc2.weight15 module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc1.weight2 module.decoder.layers.31.mlp.experts.linear_fc1.weight4 module.decoder.layers.31.mlp.experts.linear_fc1.weight6 module.decoder.layers.31.mlp.experts.linear_fc1.weight8 module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc2.weight12 + module.decoder.layers.31.mlp.experts.linear_fc2.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight7 + module.decoder.layers.31.mlp.experts.linear_fc1.weight9 Params for bucket 32 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 module.decoder.layers.30.mlp.experts.linear_fc2.weight3 module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 module.decoder.layers.30.mlp.experts.linear_fc2.weight0 module.decoder.layers.30.mlp.experts.linear_fc2.weight8 - module.decoder.layers.31.mlp.experts.linear_fc1.weight15 module.decoder.layers.30.mlp.experts.linear_fc2.weight5 module.decoder.layers.30.mlp.experts.linear_fc2.weight13 - module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 module.decoder.layers.30.mlp.experts.linear_fc2.weight2 module.decoder.layers.30.mlp.experts.linear_fc2.weight10 module.decoder.layers.30.mlp.experts.linear_fc2.weight7 module.decoder.layers.30.mlp.experts.linear_fc2.weight15 - module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 module.decoder.layers.30.mlp.experts.linear_fc2.weight4 module.decoder.layers.30.mlp.experts.linear_fc2.weight12 - module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 module.decoder.layers.30.mlp.experts.linear_fc2.weight1 module.decoder.layers.30.mlp.experts.linear_fc2.weight9 module.decoder.layers.30.mlp.experts.linear_fc2.weight6 @@ -1649,38 +1649,38 @@ Params for bucket 36 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc1.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight1 module.decoder.layers.28.mlp.experts.linear_fc2.weight9 - module.decoder.layers.28.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight6 module.decoder.layers.28.mlp.experts.linear_fc2.weight14 module.decoder.layers.28.mlp.experts.linear_fc2.weight3 module.decoder.layers.28.mlp.experts.linear_fc2.weight11 - module.decoder.layers.28.mlp.experts.linear_fc1.weight2 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 module.decoder.layers.28.mlp.experts.linear_fc2.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight8 module.decoder.layers.28.mlp.experts.linear_fc2.weight5 module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 module.decoder.layers.28.mlp.experts.linear_fc2.weight2 module.decoder.layers.28.mlp.experts.linear_fc2.weight10 - module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight7 module.decoder.layers.28.mlp.experts.linear_fc2.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight4 module.decoder.layers.28.mlp.experts.linear_fc2.weight12 - module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight2 Params for bucket 37 (40894464 elements, 40894464 padded size): - module.decoder.layers.28.mlp.experts.linear_fc1.weight4 - module.decoder.layers.28.mlp.experts.linear_fc1.weight6 - module.decoder.layers.28.mlp.experts.linear_fc1.weight8 - module.decoder.layers.28.mlp.experts.linear_fc1.weight10 - module.decoder.layers.28.mlp.experts.linear_fc1.weight12 - module.decoder.layers.28.mlp.experts.linear_fc1.weight14 - module.decoder.layers.27.mlp.experts.linear_fc2.weight1 - module.decoder.layers.28.mlp.experts.linear_fc1.weight5 module.decoder.layers.28.mlp.experts.linear_fc1.weight7 + module.decoder.layers.28.mlp.experts.linear_fc1.weight5 module.decoder.layers.28.mlp.experts.linear_fc1.weight9 module.decoder.layers.28.mlp.experts.linear_fc1.weight11 module.decoder.layers.28.mlp.experts.linear_fc1.weight13 module.decoder.layers.28.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight1 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 module.decoder.layers.27.mlp.experts.linear_fc2.weight0 Params for bucket 38 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc2.weight9 @@ -1709,27 +1709,18 @@ Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc1.weight8 module.decoder.layers.27.mlp.experts.linear_fc1.weight12 module.decoder.layers.27.mlp.experts.linear_fc1.weight14 - module.decoder.layers.26.mlp.experts.linear_fc2.weight1 - module.decoder.layers.26.mlp.experts.linear_fc2.weight3 - module.decoder.layers.26.mlp.experts.linear_fc2.weight5 + module.decoder.layers.26.mlp.experts.linear_fc2.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 module.decoder.layers.27.mlp.experts.linear_fc1.weight9 module.decoder.layers.27.mlp.experts.linear_fc1.weight7 module.decoder.layers.27.mlp.experts.linear_fc1.weight11 module.decoder.layers.27.mlp.experts.linear_fc1.weight13 - module.decoder.layers.27.mlp.experts.linear_fc1.weight15 - module.decoder.layers.26.mlp.experts.linear_fc2.weight0 - module.decoder.layers.26.mlp.experts.linear_fc2.weight2 - module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc2.weight3 + module.decoder.layers.26.mlp.experts.linear_fc2.weight5 Params for bucket 40 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc2.weight7 - module.decoder.layers.26.mlp.experts.linear_fc2.weight9 - module.decoder.layers.26.mlp.experts.linear_fc2.weight11 - module.decoder.layers.26.mlp.experts.linear_fc2.weight13 - module.decoder.layers.26.mlp.experts.linear_fc2.weight15 - module.decoder.layers.26.mlp.experts.linear_fc1.weight0 - module.decoder.layers.26.mlp.experts.linear_fc1.weight2 - module.decoder.layers.26.mlp.experts.linear_fc1.weight4 - module.decoder.layers.26.mlp.experts.linear_fc1.weight6 module.decoder.layers.26.mlp.experts.linear_fc2.weight6 module.decoder.layers.26.mlp.experts.linear_fc2.weight8 module.decoder.layers.26.mlp.experts.linear_fc2.weight10 @@ -1739,18 +1730,18 @@ Params for bucket 40 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc1.weight3 module.decoder.layers.26.mlp.experts.linear_fc1.weight5 module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight4 + module.decoder.layers.26.mlp.experts.linear_fc1.weight6 Params for bucket 41 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc1.weight8 - module.decoder.layers.26.mlp.experts.linear_fc1.weight10 - module.decoder.layers.26.mlp.experts.linear_fc1.weight12 - module.decoder.layers.26.mlp.experts.linear_fc1.weight14 - module.decoder.layers.25.mlp.experts.linear_fc2.weight0 - module.decoder.layers.25.mlp.experts.linear_fc2.weight2 - module.decoder.layers.25.mlp.experts.linear_fc2.weight4 - module.decoder.layers.25.mlp.experts.linear_fc2.weight6 - module.decoder.layers.25.mlp.experts.linear_fc2.weight8 - module.decoder.layers.26.mlp.experts.linear_fc1.weight11 module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 module.decoder.layers.26.mlp.experts.linear_fc1.weight13 module.decoder.layers.26.mlp.experts.linear_fc1.weight15 module.decoder.layers.25.mlp.experts.linear_fc2.weight1 @@ -1758,15 +1749,16 @@ Params for bucket 41 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc2.weight5 module.decoder.layers.25.mlp.experts.linear_fc2.weight7 module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 + module.decoder.layers.26.mlp.experts.linear_fc1.weight14 + module.decoder.layers.25.mlp.experts.linear_fc2.weight0 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc2.weight8 Params for bucket 42 (40894464 elements, 40894464 padded size): - module.decoder.layers.25.mlp.experts.linear_fc2.weight10 - module.decoder.layers.25.mlp.experts.linear_fc2.weight14 - module.decoder.layers.25.mlp.experts.linear_fc2.weight12 - module.decoder.layers.25.mlp.experts.linear_fc1.weight1 - module.decoder.layers.25.mlp.experts.linear_fc1.weight3 - module.decoder.layers.25.mlp.experts.linear_fc1.weight5 - module.decoder.layers.25.mlp.experts.linear_fc1.weight7 - module.decoder.layers.25.mlp.experts.linear_fc1.weight9 module.decoder.layers.25.mlp.experts.linear_fc2.weight11 module.decoder.layers.25.mlp.experts.linear_fc2.weight13 module.decoder.layers.25.mlp.experts.linear_fc2.weight15 @@ -1775,10 +1767,26 @@ Params for bucket 42 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc1.weight4 module.decoder.layers.25.mlp.experts.linear_fc1.weight6 module.decoder.layers.25.mlp.experts.linear_fc1.weight8 + module.decoder.layers.25.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 Params for bucket 43 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight8 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 module.decoder.layers.25.mlp.experts.linear_fc1.weight13 module.decoder.layers.24.mlp.experts.linear_fc2.weight7 module.decoder.layers.25.mlp.experts.linear_fc1.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight0 module.decoder.layers.24.mlp.experts.linear_fc2.weight4 module.decoder.layers.24.mlp.experts.linear_fc2.weight12 module.decoder.layers.25.mlp.experts.linear_fc1.weight15 @@ -1788,35 +1796,22 @@ Params for bucket 43 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc2.weight6 module.decoder.layers.24.mlp.experts.linear_fc2.weight3 module.decoder.layers.24.mlp.experts.linear_fc2.weight11 - module.decoder.layers.25.mlp.experts.linear_fc1.weight14 - module.decoder.layers.24.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc2.weight8 - module.decoder.layers.25.mlp.experts.linear_fc1.weight11 - module.decoder.layers.24.mlp.experts.linear_fc2.weight5 - module.decoder.layers.24.mlp.experts.linear_fc2.weight13 - module.decoder.layers.24.mlp.experts.linear_fc2.weight2 - module.decoder.layers.24.mlp.experts.linear_fc2.weight10 Params for bucket 44 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight14 module.decoder.layers.24.mlp.experts.linear_fc1.weight1 module.decoder.layers.24.mlp.experts.linear_fc1.weight3 module.decoder.layers.24.mlp.experts.linear_fc1.weight5 module.decoder.layers.24.mlp.experts.linear_fc1.weight7 module.decoder.layers.24.mlp.experts.linear_fc1.weight9 module.decoder.layers.24.mlp.experts.linear_fc1.weight11 - module.decoder.layers.24.mlp.experts.linear_fc2.weight14 - module.decoder.layers.24.mlp.experts.linear_fc1.weight0 module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 module.decoder.layers.24.mlp.experts.linear_fc1.weight4 module.decoder.layers.24.mlp.experts.linear_fc1.weight6 module.decoder.layers.24.mlp.experts.linear_fc1.weight8 module.decoder.layers.24.mlp.experts.linear_fc1.weight10 Params for bucket 45 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc1.weight13 - module.decoder.layers.23.mlp.experts.linear_fc2.weight7 - module.decoder.layers.23.mlp.experts.linear_fc2.weight15 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight12 module.decoder.layers.24.mlp.experts.linear_fc1.weight15 module.decoder.layers.23.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight9 @@ -1826,34 +1821,33 @@ Params for bucket 45 (40894464 elements, 40894464 padded size): module.decoder.layers.23.mlp.experts.linear_fc2.weight14 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight11 - module.decoder.layers.23.mlp.experts.linear_fc2.weight0 module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 module.decoder.layers.23.mlp.experts.linear_fc2.weight8 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight13 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight15 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight12 Params for bucket 46 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc1.weight8 module.decoder.layers.23.mlp.experts.linear_fc1.weight10 module.decoder.layers.23.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight7 module.decoder.layers.23.mlp.experts.linear_fc1.weight9 module.decoder.layers.23.mlp.experts.linear_fc1.weight11 module.decoder.layers.23.mlp.experts.linear_fc1.weight13 Params for bucket 47 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight14 - module.decoder.layers.22.mlp.experts.linear_fc2.weight0 - module.decoder.layers.22.mlp.experts.linear_fc2.weight8 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight13 - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight10 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 @@ -1861,18 +1855,24 @@ Params for bucket 47 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 module.decoder.layers.22.mlp.experts.linear_fc2.weight12 - module.decoder.layers.23.mlp.experts.linear_fc1.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight9 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight14 module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight14 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 Params for bucket 48 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight9 module.decoder.layers.22.mlp.experts.linear_fc1.weight11 module.decoder.layers.22.mlp.experts.linear_fc1.weight13 @@ -1884,11 +1884,6 @@ Params for bucket 48 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight12 module.decoder.layers.22.mlp.experts.linear_fc1.weight14 Params for bucket 49 (40894464 elements, 40894464 padded size): - module.decoder.layers.21.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc2.weight10 - module.decoder.layers.21.mlp.experts.linear_fc2.weight7 - module.decoder.layers.21.mlp.experts.linear_fc2.weight15 - module.decoder.layers.21.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 module.decoder.layers.21.mlp.experts.linear_fc2.weight12 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 @@ -1905,27 +1900,28 @@ Params for bucket 49 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight13 module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight10 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight15 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 Params for bucket 50 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight5 - module.decoder.layers.21.mlp.experts.linear_fc1.weight9 module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 module.decoder.layers.21.mlp.experts.linear_fc1.weight11 module.decoder.layers.21.mlp.experts.linear_fc1.weight13 module.decoder.layers.21.mlp.experts.linear_fc1.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc1.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight10 module.decoder.layers.21.mlp.experts.linear_fc1.weight12 module.decoder.layers.21.mlp.experts.linear_fc1.weight14 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 Params for bucket 51 (40894464 elements, 40894464 padded size): - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc2.weight12 - module.decoder.layers.20.mlp.experts.linear_fc2.weight9 - module.decoder.layers.20.mlp.experts.linear_fc1.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc2.weight14 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 @@ -1941,6 +1937,10 @@ Params for bucket 51 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight15 module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight7 module.decoder.layers.20.mlp.experts.linear_fc1.weight9 @@ -1960,8 +1960,8 @@ Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 53 (40894464 elements, 40894464 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight11 module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 module.decoder.layers.19.mlp.experts.linear_fc2.weight13 module.decoder.layers.19.mlp.experts.linear_fc2.weight15 module.decoder.layers.19.mlp.experts.linear_fc1.weight0 @@ -1978,10 +1978,6 @@ Params for bucket 53 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.19.mlp.experts.linear_fc1.weight7 Params for bucket 54 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc1.weight13 - module.decoder.layers.18.mlp.experts.linear_fc2.weight7 module.decoder.layers.19.mlp.experts.linear_fc1.weight10 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight15 @@ -1997,6 +1993,10 @@ Params for bucket 54 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc2.weight8 module.decoder.layers.19.mlp.experts.linear_fc1.weight11 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 Params for bucket 55 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc2.weight12 module.decoder.layers.18.mlp.experts.linear_fc2.weight14 @@ -2014,28 +2014,34 @@ Params for bucket 55 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight8 module.decoder.layers.18.mlp.experts.linear_fc1.weight10 Params for bucket 56 (40894464 elements, 40894464 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight11 - module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight8 - module.decoder.layers.18.mlp.experts.linear_fc1.weight11 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight13 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight10 - module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight15 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight12 - module.decoder.layers.18.mlp.experts.linear_fc1.weight15 - module.decoder.layers.17.mlp.experts.linear_fc2.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight9 - module.decoder.layers.18.mlp.experts.linear_fc1.weight12 - module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc2.weight14 Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 @@ -2043,13 +2049,15 @@ Params for bucket 57 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight8 module.decoder.layers.17.mlp.experts.linear_fc1.weight10 module.decoder.layers.17.mlp.experts.linear_fc1.weight12 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight3 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc1.weight9 - module.decoder.layers.17.mlp.experts.linear_fc1.weight11 Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight10 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight15 module.decoder.layers.17.mlp.experts.linear_fc1.weight14 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight12 @@ -2063,72 +2071,58 @@ Params for bucket 58 (40894464 elements, 40894464 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight11 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 module.decoder.layers.16.mlp.experts.linear_fc2.weight8 - module.decoder.layers.17.mlp.experts.linear_fc1.weight15 - module.decoder.layers.16.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc2.weight13 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 - module.decoder.layers.16.mlp.experts.linear_fc2.weight2 - module.decoder.layers.16.mlp.experts.linear_fc2.weight10 - module.decoder.layers.16.mlp.experts.linear_fc2.weight7 - module.decoder.layers.16.mlp.experts.linear_fc2.weight15 Params for bucket 59 (40894464 elements, 40894464 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight7 - module.decoder.layers.16.mlp.experts.linear_fc1.weight9 - module.decoder.layers.16.mlp.experts.linear_fc1.weight11 - module.decoder.layers.16.mlp.experts.linear_fc1.weight13 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.16.mlp.experts.linear_fc1.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight8 module.decoder.layers.16.mlp.experts.linear_fc1.weight10 module.decoder.layers.16.mlp.experts.linear_fc1.weight12 module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 Params for bucket 60 (40894464 elements, 40894464 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight15 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight10 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight15 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 module.decoder.layers.15.mlp.experts.linear_fc2.weight12 module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc2.weight9 - module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc2.weight14 - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight11 - module.decoder.layers.15.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc2.weight8 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 Params for bucket 61 (40894464 elements, 40894464 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight9 module.decoder.layers.15.mlp.experts.linear_fc1.weight11 module.decoder.layers.15.mlp.experts.linear_fc1.weight13 module.decoder.layers.15.mlp.experts.linear_fc1.weight15 module.decoder.layers.14.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 - module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight10 module.decoder.layers.15.mlp.experts.linear_fc1.weight12 module.decoder.layers.15.mlp.experts.linear_fc1.weight14 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 Params for bucket 62 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight5 - module.decoder.layers.14.mlp.experts.linear_fc2.weight13 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight10 - module.decoder.layers.14.mlp.experts.linear_fc1.weight5 module.decoder.layers.14.mlp.experts.linear_fc2.weight7 module.decoder.layers.14.mlp.experts.linear_fc2.weight15 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 @@ -2143,24 +2137,39 @@ Params for bucket 62 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight11 module.decoder.layers.14.mlp.experts.linear_fc2.weight8 module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 Params for bucket 63 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight8 module.decoder.layers.14.mlp.experts.linear_fc1.weight10 module.decoder.layers.14.mlp.experts.linear_fc1.weight12 module.decoder.layers.14.mlp.experts.linear_fc1.weight14 - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight11 module.decoder.layers.14.mlp.experts.linear_fc1.weight13 module.decoder.layers.14.mlp.experts.linear_fc1.weight15 - module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 Params for bucket 64 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight8 module.decoder.layers.13.mlp.experts.linear_fc2.weight10 @@ -2170,37 +2179,28 @@ Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight3 module.decoder.layers.13.mlp.experts.linear_fc1.weight5 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 - module.decoder.layers.13.mlp.experts.linear_fc2.weight7 - module.decoder.layers.13.mlp.experts.linear_fc2.weight9 - module.decoder.layers.13.mlp.experts.linear_fc2.weight11 - module.decoder.layers.13.mlp.experts.linear_fc2.weight13 - module.decoder.layers.13.mlp.experts.linear_fc2.weight15 - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 Params for bucket 65 (40894464 elements, 40894464 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight11 - module.decoder.layers.13.mlp.experts.linear_fc1.weight9 - module.decoder.layers.13.mlp.experts.linear_fc1.weight13 - module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 module.decoder.layers.12.mlp.experts.linear_fc2.weight1 module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc2.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight7 module.decoder.layers.12.mlp.experts.linear_fc2.weight9 - module.decoder.layers.13.mlp.experts.linear_fc1.weight8 - module.decoder.layers.13.mlp.experts.linear_fc1.weight12 - module.decoder.layers.13.mlp.experts.linear_fc1.weight10 - module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight4 module.decoder.layers.12.mlp.experts.linear_fc2.weight6 module.decoder.layers.12.mlp.experts.linear_fc2.weight8 Params for bucket 66 (40894464 elements, 40894464 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight11 module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 module.decoder.layers.12.mlp.experts.linear_fc2.weight15 module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.12.mlp.experts.linear_fc1.weight3 @@ -2216,10 +2216,6 @@ Params for bucket 66 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight8 Params for bucket 67 (40894464 elements, 40894464 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight11 - module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight9 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 @@ -2236,10 +2232,14 @@ Params for bucket 67 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight8 @@ -2252,42 +2252,50 @@ Params for bucket 68 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight11 Params for bucket 69 (40894464 elements, 40894464 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 - module.decoder.layers.11.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight11 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight15 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight9 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight8 - module.decoder.layers.10.mlp.experts.linear_fc1.weight10 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 Params for bucket 71 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 @@ -2301,21 +2309,7 @@ Params for bucket 71 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight14 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 Params for bucket 72 (40894464 elements, 40894464 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 - module.decoder.layers.9.mlp.experts.linear_fc1.weight8 - module.decoder.layers.9.mlp.experts.linear_fc1.weight10 - module.decoder.layers.9.mlp.experts.linear_fc1.weight12 - module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 @@ -2323,7 +2317,21 @@ Params for bucket 72 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 Params for bucket 73 (40894464 elements, 40894464 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 @@ -2337,23 +2345,7 @@ Params for bucket 73 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight14 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 Params for bucket 74 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight9 - module.decoder.layers.8.mlp.experts.linear_fc1.weight11 - module.decoder.layers.8.mlp.experts.linear_fc1.weight13 - module.decoder.layers.8.mlp.experts.linear_fc1.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 @@ -2361,7 +2353,23 @@ Params for bucket 74 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 Params for bucket 75 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 @@ -2373,56 +2381,43 @@ Params for bucket 75 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 76 (40894464 elements, 40894464 padded size): - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 - module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 - module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 77 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight8 - module.decoder.layers.6.mlp.experts.linear_fc2.weight9 - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 78 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight12 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight11 @@ -2437,6 +2432,11 @@ Params for bucket 78 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight12 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 @@ -2454,11 +2454,6 @@ Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight10 Params for bucket 80 (40894464 elements, 40894464 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight12 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 @@ -2475,9 +2470,14 @@ Params for bucket 80 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight10 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 Params for bucket 81 (40894464 elements, 40894464 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight9 @@ -2490,11 +2490,6 @@ Params for bucket 81 (40894464 elements, 40894464 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 Params for bucket 82 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 @@ -2511,22 +2506,26 @@ Params for bucket 82 (40894464 elements, 40894464 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 Params for bucket 83 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.3.mlp.experts.linear_fc1.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 module.decoder.layers.3.mlp.experts.linear_fc1.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 module.decoder.layers.3.mlp.experts.linear_fc1.weight14 Params for bucket 84 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 @@ -2542,6 +2541,7 @@ Params for bucket 84 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 @@ -2563,11 +2563,6 @@ Params for bucket 85 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight15 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 Params for bucket 86 (40894464 elements, 40894464 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 @@ -2583,9 +2578,14 @@ Params for bucket 86 (40894464 elements, 40894464 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 Params for bucket 87 (40894464 elements, 40894464 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 module.decoder.layers.1.mlp.experts.linear_fc1.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 @@ -2593,8 +2593,8 @@ Params for bucket 87 (40894464 elements, 40894464 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 module.decoder.layers.1.mlp.experts.linear_fc1.weight12 module.decoder.layers.1.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 @@ -2610,8 +2610,8 @@ Params for bucket 88 (40894464 elements, 40894464 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 module.decoder.layers.0.mlp.experts.linear_fc2.weight15 @@ -2621,16 +2621,16 @@ Params for bucket 88 (40894464 elements, 40894464 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 89 (25165824 elements, 25165824 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight8 - module.decoder.layers.0.mlp.experts.linear_fc1.weight12 - module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:50:36 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 01:36:46 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -2638,7 +2638,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -2651,14 +2651,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:50:36 +[after dataloaders are built] datetime: 2026-08-30 01:36:46 done with setup ... -training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (5162.77, 5166.08) - train/valid/test-data-iterators-setup ..........: (64.47, 73.98) + model-and-optimizer-setup ......................: (5228.68, 5232.56) + train/valid/test-data-iterators-setup ..........: (64.23, 71.53) +training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-29 23:50:36 +[before the start of training step] datetime: 2026-08-30 01:36:46 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 sm_count : num_sms=32, num_sms_dedup=148 @@ -2670,36 +2670,37 @@ INFO:moonep.api:MoonEP comm buffer sizes: [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1280, 1024, 256, 640, 512, 3328, 256, 1408, 896, 1408, 128, 1152, 3712, 1024, 1792, 12544, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [123, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1024, 128, 896, 128, 640, 0, 768, 128, 128, 256, 256, 256, 384, 128, 3584, 1152, 23424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[66, 74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5248, 2176, 1792, 4224, 128, 384, 128, 128, 1280, 640, 2816, 3200, 128, 128, 4224, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, 80, 89, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] - [2026-08-29 23:50:59] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22741.6 | throughput per GPU (TFLOP/s/GPU): 3.9 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.356 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:08] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22303.2 | throughput per GPU (TFLOP/s/GPU): 4.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.353 | number of skipped iterations: 0 | number of nan iterations: 0 | Number of parameters in transformer block in billions: 29.90 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 30.10 Number of parameters in most loaded shard in billions: 30.1034 Theoretical memory footprints: weight and optimizer=215316.59 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 66603.12646484375 | max allocated: 66603.126953125 | reserved: 68288.0 | max reserved: 68288.0 - [2026-08-29 23:51:00] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 788.6 | throughput per GPU (TFLOP/s/GPU): 113.0 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.499 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:00] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 651.9 | throughput per GPU (TFLOP/s/GPU): 136.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008803E+01 | loss scale: 1.0 | grad norm: 181.179 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:09] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 747.3 | throughput per GPU (TFLOP/s/GPU): 119.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.506 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:10] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 629.7 | throughput per GPU (TFLOP/s/GPU): 141.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008695E+01 | loss scale: 1.0 | grad norm: 180.332 | number of skipped iterations: 0 | number of nan iterations: 0 | Capture CUDA graph for training!!! CUDA graph capture done!!! - [2026-08-29 23:51:02] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1813.4 | throughput per GPU (TFLOP/s/GPU): 49.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604629E+00 | loss scale: 1.0 | grad norm: 8.726 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:03] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 363.1 | throughput per GPU (TFLOP/s/GPU): 245.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307946E+00 | loss scale: 1.0 | grad norm: 5.244 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:03] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 361.1 | throughput per GPU (TFLOP/s/GPU): 246.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.201571E+00 | loss scale: 1.0 | grad norm: 4.265 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:03] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 359.6 | throughput per GPU (TFLOP/s/GPU): 247.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.045153E+00 | loss scale: 1.0 | grad norm: 3.412 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:04] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 359.2 | throughput per GPU (TFLOP/s/GPU): 248.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.864072E+00 | loss scale: 1.0 | grad norm: 3.288 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:04] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 361.1 | throughput per GPU (TFLOP/s/GPU): 246.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.805015E+00 | loss scale: 1.0 | grad norm: 3.054 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:04] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 361.4 | throughput per GPU (TFLOP/s/GPU): 246.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.765068E+00 | loss scale: 1.0 | grad norm: 3.059 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:05] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 360.4 | throughput per GPU (TFLOP/s/GPU): 247.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.756466E+00 | loss scale: 1.0 | grad norm: 2.904 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:51:05] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 362.9 | throughput per GPU (TFLOP/s/GPU): 245.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.730773E+00 | loss scale: 1.0 | grad norm: 2.771 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:51:05 -W0829 23:51:06.486000 247706 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:51:06.486000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247897 closing signal SIGTERM -W0829 23:51:06.487000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247898 closing signal SIGTERM -W0829 23:51:06.488000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247899 closing signal SIGTERM -W0829 23:51:06.490000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247900 closing signal SIGTERM -W0829 23:51:06.491000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247901 closing signal SIGTERM -W0829 23:51:06.492000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247902 closing signal SIGTERM -W0829 23:51:06.493000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247903 closing signal SIGTERM -W0829 23:51:06.494000 247706 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 247904 closing signal SIGTERM + [2026-08-30 01:37:11] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1715.9 | throughput per GPU (TFLOP/s/GPU): 51.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604200E+00 | loss scale: 1.0 | grad norm: 8.755 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:12] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 356.6 | throughput per GPU (TFLOP/s/GPU): 249.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307413E+00 | loss scale: 1.0 | grad norm: 6.685 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:12] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 358.0 | throughput per GPU (TFLOP/s/GPU): 249.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.152184E+00 | loss scale: 1.0 | grad norm: 3.088 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:12] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 354.0 | throughput per GPU (TFLOP/s/GPU): 251.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.998609E+00 | loss scale: 1.0 | grad norm: 3.401 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:13] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 352.2 | throughput per GPU (TFLOP/s/GPU): 253.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.841763E+00 | loss scale: 1.0 | grad norm: 3.072 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:13] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 354.1 | throughput per GPU (TFLOP/s/GPU): 251.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.792585E+00 | loss scale: 1.0 | grad norm: 2.781 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:14] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 353.5 | throughput per GPU (TFLOP/s/GPU): 252.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.753055E+00 | loss scale: 1.0 | grad norm: 2.742 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:14] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 356.0 | throughput per GPU (TFLOP/s/GPU): 250.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.745409E+00 | loss scale: 1.0 | grad norm: 2.560 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:37:14] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 353.7 | throughput per GPU (TFLOP/s/GPU): 251.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.723271E+00 | loss scale: 1.0 | grad norm: 2.440 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 01:37:14 +[rank0]:[W830 01:37:16.306690296 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 01:37:17.222000 390312 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 01:37:17.222000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390490 closing signal SIGTERM +W0830 01:37:17.223000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390491 closing signal SIGTERM +W0830 01:37:17.224000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390492 closing signal SIGTERM +W0830 01:37:17.225000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390493 closing signal SIGTERM +W0830 01:37:17.226000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390494 closing signal SIGTERM +W0830 01:37:17.227000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390495 closing signal SIGTERM +W0830 01:37:17.227000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390496 closing signal SIGTERM +W0830 01:37:17.229000 390312 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 390497 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -2727,4 +2728,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 247706 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 390312 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/mem_samples.csv index aba3db9e90f4a12d3dfb714a5b12f4809037fcf0..b20413e6bf4901abf300402c22558edc195b2197 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/mem_samples.csv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/mem_samples.csv @@ -26,7 +26,7 @@ 1, 1 2, 1 3, 1 -4, 3 +4, 1 5, 4 6, 4 7, 4 @@ -64,20 +64,20 @@ 7, 4 0, 4 1, 629 -2, 629 -3, 629 +2, 45 +3, 35 4, 629 5, 629 6, 629 7, 629 -0, 929 -1, 935 -2, 907 -3, 923 -4, 943 -5, 921 -6, 921 -7, 947 +0, 875 +1, 875 +2, 877 +3, 893 +4, 903 +5, 905 +6, 899 +7, 909 0, 1831 1, 1831 2, 1831 @@ -86,14 +86,14 @@ 5, 1831 6, 1831 7, 1831 -0, 15033 -1, 15189 -2, 15209 -3, 15169 -4, 15309 -5, 14441 -6, 14657 -7, 15229 +0, 16477 +1, 16833 +2, 17345 +3, 16517 +4, 17209 +5, 16913 +6, 16773 +7, 16577 0, 47207 1, 47207 2, 47207 @@ -110,54 +110,54 @@ 5, 47551 6, 47551 7, 47551 -0, 48133 -1, 47957 -2, 48151 +0, 48207 +1, 47959 +2, 48235 3, 47959 -4, 47959 -5, 48161 +4, 47999 +5, 48235 6, 47959 -7, 48161 +7, 48331 0, 49103 -1, 48761 -2, 49131 +1, 48759 +2, 49133 3, 48759 -4, 48897 -5, 49133 -6, 48761 -7, 49131 +4, 48895 +5, 49131 +6, 48759 +7, 49133 0, 49249 1, 48907 -2, 49277 +2, 49279 3, 48905 -4, 49043 -5, 49279 -6, 48907 -7, 49277 +4, 49041 +5, 49277 +6, 48905 +7, 49279 0, 49251 1, 48907 -2, 49277 +2, 49279 3, 48907 -4, 49043 +4, 49041 5, 49279 6, 48907 7, 49279 -0, 63573 -1, 63551 -2, 63547 -3, 63557 -4, 63563 -5, 63547 -6, 63557 -7, 63547 -0, 83465 -1, 83459 -2, 83473 -3, 83455 -4, 83517 -5, 83465 -6, 83477 -7, 83473 +0, 66095 +1, 66077 +2, 65899 +3, 65887 +4, 65957 +5, 65919 +6, 65909 +7, 65899 +0, 83463 +1, 83509 +2, 83483 +3, 83467 +4, 83535 +5, 83455 +6, 83487 +7, 83515 0, 83947 1, 83951 2, 83941 @@ -166,67 +166,67 @@ 5, 83941 6, 83931 7, 83941 -0, 84337 -1, 84341 -2, 84331 -3, 84321 -4, 84349 -5, 84331 -6, 84321 -7, 84331 -0, 84997 -1, 85001 -2, 84991 -3, 84981 -4, 85009 -5, 84991 -6, 84981 -7, 84991 -0, 95527 -1, 95431 -2, 95503 -3, 95295 -4, 95471 -5, 95577 -6, 95423 -7, 95537 -0, 103155 -1, 106155 -2, 103043 -3, 102399 -4, 102523 -5, 103443 -6, 102351 -7, 102643 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116555 -7, 116701 -0, 116703 -1, 116611 -2, 116701 -3, 116459 -4, 116583 -5, 116701 -6, 116555 -7, 116701 -0, 23094 -1, 116609 -2, 116699 -3, 116457 -4, 116581 -5, 116699 -6, 116553 -7, 116699 -0, 21491 -1, 116609 -2, 22774 -3, 21174 -4, 21174 -5, 21174 -6, 21174 -7, 21174 +0, 84249 +1, 84237 +2, 84221 +3, 84217 +4, 84273 +5, 84265 +6, 84201 +7, 84257 +0, 89637 +1, 89977 +2, 90223 +3, 90153 +4, 89697 +5, 90063 +6, 89849 +7, 89919 +0, 95507 +1, 95411 +2, 95461 +3, 95249 +4, 95409 +5, 95503 +6, 95353 +7, 95529 +0, 107557 +1, 108155 +2, 111849 +3, 106913 +4, 107327 +5, 107955 +6, 114103 +7, 107845 +0, 116707 +1, 116615 +2, 116705 +3, 116463 +4, 116587 +5, 116705 +6, 116559 +7, 116705 +0, 116707 +1, 116615 +2, 116705 +3, 116463 +4, 116587 +5, 116705 +6, 116559 +7, 116705 +0, 116061 +1, 115969 +2, 116059 +3, 115817 +4, 115941 +5, 116059 +6, 115913 +7, 116059 +0, 20737 +1, 20740 +2, 20740 +3, 20740 +4, 20996 +5, 116057 +6, 22020 +7, 20740 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/run.env b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/run.env index 4d121388cdc2db102c1783c999c4de53c0e99bb8..2e2a2fd96c750ea9322cc0a5bc54f2c95e45a4b6 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/run.env +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/run.env @@ -3,7 +3,7 @@ config=QWEN3_30B family=tierB repo=/workspace/Megatron-MoE-Echo branch=moonep_backend_async_grad_reduce -commit=7bc235bf +commit=2e68c8ad moonep_source_dir=/workspace/MoonEP moonep_source_branch=megatron-ddp-symmetric-params moonep_source_commit=d8bc666 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/train.log b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/train.log index 6d694271ed9613ddc3c50e49b70c471acd22b3af..6056643df6f3aa2eb1da4c2971ccb3328bfeedc4 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/train.log +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/train.log @@ -1,16 +1,22 @@ -[23:51:22] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +[01:37:32] WARN: moonep_backend_async_grad_reduce lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) ============================================================== QWEN3_30B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=768 m=16 L=48 seq=4096 gbs=8 (1 microbatches) cap_mode=none cap=none cg=1 recompute=none memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3/memory_snapshots/snapshot.pickle - repo=/workspace/Megatron-MoE-Echo @ 7bc235bf + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad ============================================================== -W0829 23:51:24.003000 250100 torch/distributed/run.py:766] -W0829 23:51:24.003000 250100 torch/distributed/run.py:766] ***************************************** -W0829 23:51:24.003000 250100 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0829 23:51:24.003000 250100 torch/distributed/run.py:766] ***************************************** +W0830 01:37:33.434000 392738 torch/distributed/run.py:766] +W0830 01:37:33.434000 392738 torch/distributed/run.py:766] ***************************************** +W0830 01:37:33.434000 392738 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 01:37:33.434000 392738 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -564,64 +570,58 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode > initializing torch distributed ... /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 76 - is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank6]:[W829 23:51:42.131819911 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W829 23:51:42.131935605 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W829 23:51:42.131978759 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W829 23:51:42.132214150 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W829 23:51:42.132276981 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W829 23:51:42.132360378 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W829 23:51:42.133805219 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W830 01:37:51.561351776 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 01:37:51.561351701 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 01:37:51.561353708 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 01:37:51.561389836 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 01:37:51.561436039 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 01:37:51.561495724 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 01:37:51.561552593 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds +>>> done with dataset index builder. Compilation time: 0.035 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W829 23:51:43.305117118 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.942 seconds -time to initialize megatron (seconds): 8.522 -[after megatron is initialized] datetime: 2026-08-29 23:51:49 +[rank0]:[W830 01:37:51.728983412 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.806 seconds +time to initialize megatron (seconds): 7.205 +[after megatron is initialized] datetime: 2026-08-30 01:37:57 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -646,111 +646,111 @@ Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (49033984 elements, 49033984 padded size): module.decoder.layers.47.pre_mlp_layernorm.weight - module.decoder.layers.46.mlp.router.weight module.decoder.layers.46.pre_mlp_layernorm.weight module.decoder.layers.47.self_attention.k_layernorm.weight + module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight module.decoder.layers.45.mlp.router.weight - module.decoder.layers.45.pre_mlp_layernorm.weight module.decoder.layers.47.self_attention.q_layernorm.weight - module.decoder.layers.47.self_attention.linear_qkv.weight module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.46.self_attention.k_layernorm.weight module.decoder.layers.46.self_attention.q_layernorm.weight module.decoder.layers.46.self_attention.linear_qkv.weight module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.47.self_attention.linear_proj.weight module.decoder.layers.46.self_attention.linear_proj.weight + module.decoder.layers.45.pre_mlp_layernorm.weight module.decoder.layers.45.self_attention.k_layernorm.weight - module.decoder.layers.45.self_attention.linear_qkv.weight - module.decoder.final_layernorm.weight module.decoder.layers.47.mlp.router.weight module.decoder.layers.45.self_attention.q_layernorm.weight + module.decoder.final_layernorm.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.45.self_attention.linear_qkv.weight Params for bucket 3 (46672384 elements, 46672384 padded size): - module.decoder.layers.44.pre_mlp_layernorm.weight - module.decoder.layers.44.mlp.router.weight + module.decoder.layers.45.self_attention.linear_proj.weight module.decoder.layers.44.self_attention.k_layernorm.weight + module.decoder.layers.44.mlp.router.weight + module.decoder.layers.44.pre_mlp_layernorm.weight module.decoder.layers.44.self_attention.q_layernorm.weight module.decoder.layers.44.self_attention.linear_qkv.weight module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.44.self_attention.linear_proj.weight module.decoder.layers.43.mlp.router.weight module.decoder.layers.43.self_attention.k_layernorm.weight - module.decoder.layers.43.self_attention.linear_qkv.weight module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.45.self_attention.linear_proj.weight module.decoder.layers.43.pre_mlp_layernorm.weight module.decoder.layers.43.self_attention.q_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.weight module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.43.self_attention.linear_proj.weight Params for bucket 4 (49031936 elements, 49031936 padded size): module.decoder.layers.42.pre_mlp_layernorm.weight module.decoder.layers.41.pre_mlp_layernorm.weight module.decoder.layers.40.mlp.router.weight + module.decoder.layers.40.self_attention.linear_qkv.weight module.decoder.layers.42.self_attention.k_layernorm.weight - module.decoder.layers.42.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.q_layernorm.weight module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.41.self_attention.k_layernorm.weight module.decoder.layers.41.self_attention.q_layernorm.weight module.decoder.layers.41.self_attention.linear_qkv.weight module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.40.pre_mlp_layernorm.weight module.decoder.layers.42.self_attention.linear_proj.weight module.decoder.layers.41.self_attention.linear_proj.weight + module.decoder.layers.40.pre_mlp_layernorm.weight module.decoder.layers.41.mlp.router.weight - module.decoder.layers.40.self_attention.linear_qkv.weight - module.decoder.layers.40.self_attention.k_layernorm.weight module.decoder.layers.42.mlp.router.weight - module.decoder.layers.40.self_attention.q_layernorm.weight + module.decoder.layers.40.self_attention.k_layernorm.weight Params for bucket 5 (46672384 elements, 46672384 padded size): module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.mlp.router.weight + module.decoder.layers.38.pre_mlp_layernorm.weight + module.decoder.layers.38.self_attention.k_layernorm.weight + module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight + module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight module.decoder.layers.39.mlp.router.weight + module.decoder.layers.40.self_attention.linear_proj.weight module.decoder.layers.39.pre_mlp_layernorm.weight module.decoder.layers.39.self_attention.k_layernorm.weight module.decoder.layers.39.self_attention.q_layernorm.weight module.decoder.layers.39.self_attention.linear_qkv.weight module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.39.self_attention.linear_proj.weight - module.decoder.layers.38.mlp.router.weight - module.decoder.layers.38.self_attention.k_layernorm.weight - module.decoder.layers.38.self_attention.linear_proj.weight - module.decoder.layers.40.self_attention.linear_proj.weight - module.decoder.layers.38.pre_mlp_layernorm.weight - module.decoder.layers.38.self_attention.q_layernorm.weight - module.decoder.layers.38.self_attention.linear_qkv.weight - module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight Params for bucket 6 (49031936 elements, 49031936 padded size): module.decoder.layers.37.pre_mlp_layernorm.weight module.decoder.layers.36.pre_mlp_layernorm.weight - module.decoder.layers.35.self_attention.linear_qkv.weight - module.decoder.layers.35.mlp.router.weight - module.decoder.layers.35.self_attention.k_layernorm.weight module.decoder.layers.37.self_attention.k_layernorm.weight - module.decoder.layers.35.self_attention.q_layernorm.weight + module.decoder.layers.36.self_attention.k_layernorm.weight + module.decoder.layers.36.self_attention.linear_qkv.weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.q_layernorm.weight module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.36.self_attention.k_layernorm.weight module.decoder.layers.36.self_attention.q_layernorm.weight - module.decoder.layers.36.self_attention.linear_qkv.weight module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.q_layernorm.weight module.decoder.layers.37.self_attention.linear_proj.weight module.decoder.layers.36.self_attention.linear_proj.weight - module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.35.mlp.router.weight module.decoder.layers.36.mlp.router.weight module.decoder.layers.37.mlp.router.weight + module.decoder.layers.35.pre_mlp_layernorm.weight Params for bucket 7 (46672384 elements, 46672384 padded size): - module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.33.mlp.router.weight module.decoder.layers.33.pre_mlp_layernorm.weight module.decoder.layers.33.self_attention.k_layernorm.weight module.decoder.layers.33.self_attention.q_layernorm.weight module.decoder.layers.33.self_attention.linear_qkv.weight module.decoder.layers.33.self_attention.linear_proj.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight module.decoder.layers.34.mlp.router.weight module.decoder.layers.35.self_attention.linear_proj.weight - module.decoder.layers.34.self_attention.k_layernorm.weight module.decoder.layers.34.self_attention.q_layernorm.weight module.decoder.layers.34.self_attention.linear_qkv.weight module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight @@ -758,24 +758,24 @@ Params for bucket 7 (46672384 elements, 46672384 padded size): module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight Params for bucket 8 (49031936 elements, 49031936 padded size): module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.mlp.router.weight module.decoder.layers.31.pre_mlp_layernorm.weight - module.decoder.layers.31.self_attention.k_layernorm.weight - module.decoder.layers.31.self_attention.linear_qkv.weight module.decoder.layers.30.self_attention.k_layernorm.weight - module.decoder.layers.30.self_attention.linear_qkv.weight module.decoder.layers.32.self_attention.linear_qkv.weight - module.decoder.layers.32.self_attention.k_layernorm.weight module.decoder.layers.32.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.31.self_attention.q_layernorm.weight - module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.30.self_attention.q_layernorm.weight module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.31.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.31.self_attention.linear_proj.weight - module.decoder.layers.30.mlp.router.weight module.decoder.layers.32.mlp.router.weight - module.decoder.layers.30.pre_mlp_layernorm.weight - module.decoder.layers.31.mlp.router.weight + module.decoder.layers.30.mlp.router.weight Params for bucket 9 (46672384 elements, 46672384 padded size): module.decoder.layers.29.mlp.router.weight module.decoder.layers.30.self_attention.linear_proj.weight @@ -799,16 +799,16 @@ Params for bucket 10 (49031936 elements, 49031936 padded size): module.decoder.layers.26.pre_mlp_layernorm.weight module.decoder.layers.25.pre_mlp_layernorm.weight module.decoder.layers.26.self_attention.k_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.k_layernorm.weight module.decoder.layers.27.self_attention.k_layernorm.weight - module.decoder.layers.27.self_attention.linear_qkv.weight module.decoder.layers.27.self_attention.q_layernorm.weight + module.decoder.layers.27.self_attention.linear_qkv.weight module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.26.self_attention.q_layernorm.weight - module.decoder.layers.26.self_attention.linear_qkv.weight - module.decoder.layers.25.self_attention.k_layernorm.weight - module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.27.self_attention.linear_proj.weight module.decoder.layers.26.self_attention.linear_proj.weight module.decoder.layers.26.mlp.router.weight @@ -823,9 +823,9 @@ Params for bucket 11 (46672384 elements, 46672384 padded size): module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.23.self_attention.linear_proj.weight module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.k_layernorm.weight module.decoder.layers.24.mlp.router.weight module.decoder.layers.25.self_attention.linear_proj.weight - module.decoder.layers.24.self_attention.k_layernorm.weight module.decoder.layers.24.self_attention.q_layernorm.weight module.decoder.layers.24.self_attention.linear_qkv.weight module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight @@ -834,32 +834,25 @@ Params for bucket 12 (49031936 elements, 49031936 padded size): module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.21.pre_mlp_layernorm.weight - module.decoder.layers.20.pre_mlp_layernorm.weight module.decoder.layers.21.self_attention.k_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight module.decoder.layers.20.self_attention.k_layernorm.weight - module.decoder.layers.22.self_attention.q_layernorm.weight module.decoder.layers.22.self_attention.k_layernorm.weight module.decoder.layers.22.self_attention.linear_qkv.weight module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.21.self_attention.q_layernorm.weight module.decoder.layers.21.self_attention.linear_qkv.weight module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.20.self_attention.linear_qkv.weight - module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.22.self_attention.q_layernorm.weight module.decoder.layers.21.self_attention.linear_proj.weight - module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.q_layernorm.weight + module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.21.mlp.router.weight Params for bucket 13 (46672384 elements, 46672384 padded size): - module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.19.self_attention.k_layernorm.weight - module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.self_attention.q_layernorm.weight - module.decoder.layers.19.self_attention.linear_qkv.weight - module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.19.self_attention.linear_proj.weight - module.decoder.layers.20.self_attention.linear_proj.weight module.decoder.layers.18.mlp.router.weight module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.self_attention.k_layernorm.weight @@ -867,146 +860,150 @@ Params for bucket 13 (46672384 elements, 46672384 padded size): module.decoder.layers.18.self_attention.linear_qkv.weight module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.19.self_attention.k_layernorm.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight Params for bucket 14 (49031936 elements, 49031936 padded size): module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.16.self_attention.k_layernorm.weight module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.k_layernorm.weight module.decoder.layers.17.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.q_layernorm.weight module.decoder.layers.16.self_attention.linear_qkv.weight - module.decoder.layers.15.self_attention.k_layernorm.weight - module.decoder.layers.15.self_attention.linear_qkv.weight module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.q_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.15.self_attention.q_layernorm.weight module.decoder.layers.17.self_attention.linear_proj.weight module.decoder.layers.16.mlp.router.weight + module.decoder.layers.15.mlp.router.weight Params for bucket 15 (46672384 elements, 46672384 padded size): - module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.k_layernorm.weight module.decoder.layers.14.self_attention.q_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.pre_mlp_layernorm.weight module.decoder.layers.13.self_attention.k_layernorm.weight module.decoder.layers.13.self_attention.q_layernorm.weight - module.decoder.layers.13.self_attention.linear_qkv.weight module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.self_attention.k_layernorm.weight - module.decoder.layers.14.mlp.router.weight module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight Params for bucket 16 (49031936 elements, 49031936 padded size): module.decoder.layers.12.mlp.router.weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.10.self_attention.k_layernorm.weight - module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight - module.decoder.layers.12.self_attention.k_layernorm.weight - module.decoder.layers.12.self_attention.linear_qkv.weight - module.decoder.layers.11.self_attention.q_layernorm.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.12.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.10.mlp.router.weight module.decoder.layers.11.mlp.router.weight Params for bucket 17 (46672384 elements, 46672384 padded size): + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_proj.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.q_layernorm.weight Params for bucket 18 (49031936 elements, 49031936 padded size): - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.7.mlp.router.weight module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.k_layernorm.weight - module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.7.self_attention.q_layernorm.weight - module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.mlp.router.weight - module.decoder.layers.6.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 19 (46672384 elements, 46672384 padded size): module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.self_attention.k_layernorm.weight - module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.4.self_attention.k_layernorm.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.4.self_attention.q_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 20 (49031936 elements, 49031936 padded size): module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.1.mlp.router.weight module.decoder.layers.0.mlp.router.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.k_layernorm.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.k_layernorm.weight - module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight Params for bucket 21 (111413248 elements, 111413248 padded size): module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.embedding.word_embeddings.weight module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 Params for bucket 1 (40894464 elements, 40894464 padded size): - module.decoder.layers.47.mlp.experts.linear_fc2.weight1 - module.decoder.layers.47.mlp.experts.linear_fc2.weight9 - module.decoder.layers.47.mlp.experts.linear_fc1.weight4 module.decoder.layers.47.mlp.experts.linear_fc2.weight6 module.decoder.layers.47.mlp.experts.linear_fc2.weight14 module.decoder.layers.47.mlp.experts.linear_fc1.weight1 @@ -1025,25 +1022,26 @@ Params for bucket 1 (40894464 elements, 40894464 padded size): module.decoder.layers.47.mlp.experts.linear_fc1.weight2 module.decoder.layers.47.mlp.experts.linear_fc2.weight4 module.decoder.layers.47.mlp.experts.linear_fc2.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight4 Params for bucket 2 (40894464 elements, 40894464 padded size): - module.decoder.layers.47.mlp.experts.linear_fc1.weight6 - module.decoder.layers.47.mlp.experts.linear_fc1.weight8 - module.decoder.layers.47.mlp.experts.linear_fc1.weight10 - module.decoder.layers.47.mlp.experts.linear_fc1.weight12 - module.decoder.layers.47.mlp.experts.linear_fc1.weight14 - module.decoder.layers.46.mlp.experts.linear_fc2.weight0 - module.decoder.layers.46.mlp.experts.linear_fc2.weight2 module.decoder.layers.47.mlp.experts.linear_fc1.weight5 - module.decoder.layers.47.mlp.experts.linear_fc1.weight9 module.decoder.layers.47.mlp.experts.linear_fc1.weight7 + module.decoder.layers.47.mlp.experts.linear_fc1.weight9 module.decoder.layers.47.mlp.experts.linear_fc1.weight11 module.decoder.layers.47.mlp.experts.linear_fc1.weight13 module.decoder.layers.47.mlp.experts.linear_fc1.weight15 module.decoder.layers.46.mlp.experts.linear_fc2.weight1 module.decoder.layers.46.mlp.experts.linear_fc2.weight3 + module.decoder.layers.47.mlp.experts.linear_fc1.weight8 + module.decoder.layers.47.mlp.experts.linear_fc1.weight6 + module.decoder.layers.47.mlp.experts.linear_fc1.weight10 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight14 + module.decoder.layers.46.mlp.experts.linear_fc2.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight2 Params for bucket 3 (40894464 elements, 40894464 padded size): - module.decoder.layers.46.mlp.experts.linear_fc2.weight10 - module.decoder.layers.46.mlp.experts.linear_fc1.weight5 module.decoder.layers.46.mlp.experts.linear_fc2.weight7 module.decoder.layers.46.mlp.experts.linear_fc2.weight15 module.decoder.layers.46.mlp.experts.linear_fc1.weight2 @@ -1061,35 +1059,37 @@ Params for bucket 3 (40894464 elements, 40894464 padded size): module.decoder.layers.46.mlp.experts.linear_fc2.weight5 module.decoder.layers.46.mlp.experts.linear_fc2.weight13 module.decoder.layers.46.mlp.experts.linear_fc1.weight0 + module.decoder.layers.46.mlp.experts.linear_fc2.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight5 Params for bucket 4 (40894464 elements, 40894464 padded size): + module.decoder.layers.46.mlp.experts.linear_fc1.weight10 + module.decoder.layers.46.mlp.experts.linear_fc1.weight8 + module.decoder.layers.46.mlp.experts.linear_fc1.weight12 + module.decoder.layers.46.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc2.weight2 + module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 module.decoder.layers.46.mlp.experts.linear_fc1.weight7 module.decoder.layers.46.mlp.experts.linear_fc1.weight11 module.decoder.layers.46.mlp.experts.linear_fc1.weight9 module.decoder.layers.46.mlp.experts.linear_fc1.weight13 module.decoder.layers.46.mlp.experts.linear_fc1.weight15 - module.decoder.layers.45.mlp.experts.linear_fc2.weight0 - module.decoder.layers.45.mlp.experts.linear_fc2.weight2 - module.decoder.layers.45.mlp.experts.linear_fc2.weight4 - module.decoder.layers.45.mlp.experts.linear_fc2.weight6 - module.decoder.layers.46.mlp.experts.linear_fc1.weight8 - module.decoder.layers.46.mlp.experts.linear_fc1.weight10 - module.decoder.layers.46.mlp.experts.linear_fc1.weight12 - module.decoder.layers.46.mlp.experts.linear_fc1.weight14 module.decoder.layers.45.mlp.experts.linear_fc2.weight1 module.decoder.layers.45.mlp.experts.linear_fc2.weight3 module.decoder.layers.45.mlp.experts.linear_fc2.weight5 module.decoder.layers.45.mlp.experts.linear_fc2.weight7 Params for bucket 5 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc2.weight8 - module.decoder.layers.45.mlp.experts.linear_fc2.weight12 module.decoder.layers.45.mlp.experts.linear_fc2.weight10 + module.decoder.layers.45.mlp.experts.linear_fc2.weight12 module.decoder.layers.45.mlp.experts.linear_fc2.weight14 module.decoder.layers.45.mlp.experts.linear_fc1.weight1 module.decoder.layers.45.mlp.experts.linear_fc1.weight3 module.decoder.layers.45.mlp.experts.linear_fc1.weight5 module.decoder.layers.45.mlp.experts.linear_fc1.weight7 - module.decoder.layers.45.mlp.experts.linear_fc2.weight9 module.decoder.layers.45.mlp.experts.linear_fc2.weight11 + module.decoder.layers.45.mlp.experts.linear_fc2.weight9 module.decoder.layers.45.mlp.experts.linear_fc2.weight13 module.decoder.layers.45.mlp.experts.linear_fc2.weight15 module.decoder.layers.45.mlp.experts.linear_fc1.weight0 @@ -1098,6 +1098,10 @@ Params for bucket 5 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc1.weight6 module.decoder.layers.45.mlp.experts.linear_fc1.weight8 Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.45.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc2.weight5 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 + module.decoder.layers.44.mlp.experts.linear_fc2.weight10 module.decoder.layers.45.mlp.experts.linear_fc1.weight13 module.decoder.layers.44.mlp.experts.linear_fc2.weight7 module.decoder.layers.45.mlp.experts.linear_fc1.weight10 @@ -1113,10 +1117,6 @@ Params for bucket 6 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc1.weight14 module.decoder.layers.44.mlp.experts.linear_fc2.weight0 module.decoder.layers.44.mlp.experts.linear_fc2.weight8 - module.decoder.layers.45.mlp.experts.linear_fc1.weight11 - module.decoder.layers.44.mlp.experts.linear_fc2.weight5 - module.decoder.layers.44.mlp.experts.linear_fc2.weight2 - module.decoder.layers.44.mlp.experts.linear_fc2.weight10 Params for bucket 7 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc2.weight15 module.decoder.layers.44.mlp.experts.linear_fc1.weight0 @@ -1134,6 +1134,11 @@ Params for bucket 7 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc1.weight7 module.decoder.layers.44.mlp.experts.linear_fc1.weight9 Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight12 + module.decoder.layers.43.mlp.experts.linear_fc2.weight6 + module.decoder.layers.43.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 module.decoder.layers.44.mlp.experts.linear_fc1.weight14 module.decoder.layers.43.mlp.experts.linear_fc2.weight0 module.decoder.layers.43.mlp.experts.linear_fc2.weight8 @@ -1150,26 +1155,26 @@ Params for bucket 8 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc1.weight15 module.decoder.layers.43.mlp.experts.linear_fc2.weight1 module.decoder.layers.43.mlp.experts.linear_fc2.weight9 - module.decoder.layers.44.mlp.experts.linear_fc1.weight12 - module.decoder.layers.43.mlp.experts.linear_fc2.weight6 - module.decoder.layers.43.mlp.experts.linear_fc2.weight14 - module.decoder.layers.43.mlp.experts.linear_fc2.weight3 - module.decoder.layers.43.mlp.experts.linear_fc2.weight11 Params for bucket 9 (40894464 elements, 40894464 padded size): - module.decoder.layers.43.mlp.experts.linear_fc1.weight1 module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 module.decoder.layers.43.mlp.experts.linear_fc1.weight5 module.decoder.layers.43.mlp.experts.linear_fc1.weight7 module.decoder.layers.43.mlp.experts.linear_fc1.weight9 module.decoder.layers.43.mlp.experts.linear_fc1.weight11 module.decoder.layers.43.mlp.experts.linear_fc1.weight0 - module.decoder.layers.43.mlp.experts.linear_fc1.weight2 module.decoder.layers.43.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 module.decoder.layers.43.mlp.experts.linear_fc1.weight6 module.decoder.layers.43.mlp.experts.linear_fc1.weight8 module.decoder.layers.43.mlp.experts.linear_fc1.weight10 module.decoder.layers.43.mlp.experts.linear_fc1.weight12 Params for bucket 10 (40894464 elements, 40894464 padded size): + module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.42.mlp.experts.linear_fc2.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight14 + module.decoder.layers.42.mlp.experts.linear_fc2.weight4 + module.decoder.layers.42.mlp.experts.linear_fc2.weight12 module.decoder.layers.42.mlp.experts.linear_fc2.weight1 module.decoder.layers.42.mlp.experts.linear_fc2.weight9 module.decoder.layers.42.mlp.experts.linear_fc2.weight6 @@ -1186,11 +1191,6 @@ Params for bucket 10 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight2 module.decoder.layers.42.mlp.experts.linear_fc2.weight10 - module.decoder.layers.42.mlp.experts.linear_fc2.weight7 - module.decoder.layers.42.mlp.experts.linear_fc2.weight15 - module.decoder.layers.43.mlp.experts.linear_fc1.weight14 - module.decoder.layers.42.mlp.experts.linear_fc2.weight4 - module.decoder.layers.42.mlp.experts.linear_fc2.weight12 Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc1.weight4 @@ -1206,6 +1206,13 @@ Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight11 module.decoder.layers.42.mlp.experts.linear_fc1.weight13 Params for bucket 12 (40894464 elements, 40894464 padded size): + module.decoder.layers.41.mlp.experts.linear_fc2.weight0 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight3 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.41.mlp.experts.linear_fc2.weight5 + module.decoder.layers.41.mlp.experts.linear_fc2.weight13 + module.decoder.layers.41.mlp.experts.linear_fc1.weight0 module.decoder.layers.41.mlp.experts.linear_fc2.weight2 module.decoder.layers.41.mlp.experts.linear_fc2.weight10 module.decoder.layers.41.mlp.experts.linear_fc2.weight7 @@ -1220,13 +1227,6 @@ Params for bucket 12 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight1 module.decoder.layers.41.mlp.experts.linear_fc2.weight3 module.decoder.layers.41.mlp.experts.linear_fc2.weight11 - module.decoder.layers.41.mlp.experts.linear_fc2.weight0 - module.decoder.layers.41.mlp.experts.linear_fc2.weight8 - module.decoder.layers.41.mlp.experts.linear_fc1.weight3 - module.decoder.layers.42.mlp.experts.linear_fc1.weight15 - module.decoder.layers.41.mlp.experts.linear_fc2.weight5 - module.decoder.layers.41.mlp.experts.linear_fc2.weight13 - module.decoder.layers.41.mlp.experts.linear_fc1.weight0 Params for bucket 13 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight7 module.decoder.layers.41.mlp.experts.linear_fc1.weight5 @@ -1243,6 +1243,11 @@ Params for bucket 13 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight14 module.decoder.layers.40.mlp.experts.linear_fc2.weight0 Params for bucket 14 (40894464 elements, 40894464 padded size): + module.decoder.layers.40.mlp.experts.linear_fc2.weight9 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc2.weight6 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.40.mlp.experts.linear_fc1.weight1 module.decoder.layers.40.mlp.experts.linear_fc2.weight3 module.decoder.layers.40.mlp.experts.linear_fc2.weight11 module.decoder.layers.40.mlp.experts.linear_fc2.weight8 @@ -1258,11 +1263,6 @@ Params for bucket 14 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc1.weight2 module.decoder.layers.40.mlp.experts.linear_fc2.weight4 module.decoder.layers.40.mlp.experts.linear_fc2.weight12 - module.decoder.layers.40.mlp.experts.linear_fc2.weight9 - module.decoder.layers.40.mlp.experts.linear_fc1.weight4 - module.decoder.layers.40.mlp.experts.linear_fc2.weight6 - module.decoder.layers.40.mlp.experts.linear_fc2.weight14 - module.decoder.layers.40.mlp.experts.linear_fc1.weight1 Params for bucket 15 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc1.weight6 module.decoder.layers.40.mlp.experts.linear_fc1.weight10 @@ -1304,21 +1304,29 @@ Params for bucket 17 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc1.weight11 module.decoder.layers.39.mlp.experts.linear_fc1.weight13 module.decoder.layers.39.mlp.experts.linear_fc1.weight15 - module.decoder.layers.38.mlp.experts.linear_fc2.weight0 - module.decoder.layers.38.mlp.experts.linear_fc2.weight2 - module.decoder.layers.38.mlp.experts.linear_fc2.weight4 - module.decoder.layers.38.mlp.experts.linear_fc2.weight6 - module.decoder.layers.38.mlp.experts.linear_fc2.weight8 - module.decoder.layers.39.mlp.experts.linear_fc1.weight8 - module.decoder.layers.39.mlp.experts.linear_fc1.weight12 - module.decoder.layers.39.mlp.experts.linear_fc1.weight10 - module.decoder.layers.39.mlp.experts.linear_fc1.weight14 module.decoder.layers.38.mlp.experts.linear_fc2.weight1 module.decoder.layers.38.mlp.experts.linear_fc2.weight3 module.decoder.layers.38.mlp.experts.linear_fc2.weight5 module.decoder.layers.38.mlp.experts.linear_fc2.weight7 module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc1.weight12 + module.decoder.layers.39.mlp.experts.linear_fc1.weight10 + module.decoder.layers.39.mlp.experts.linear_fc1.weight14 + module.decoder.layers.38.mlp.experts.linear_fc2.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc2.weight8 Params for bucket 18 (40894464 elements, 40894464 padded size): + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.38.mlp.experts.linear_fc1.weight4 + module.decoder.layers.38.mlp.experts.linear_fc1.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight8 module.decoder.layers.38.mlp.experts.linear_fc2.weight10 module.decoder.layers.38.mlp.experts.linear_fc2.weight14 module.decoder.layers.38.mlp.experts.linear_fc2.weight12 @@ -1327,23 +1335,7 @@ Params for bucket 18 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc1.weight5 module.decoder.layers.38.mlp.experts.linear_fc1.weight7 module.decoder.layers.38.mlp.experts.linear_fc1.weight9 - module.decoder.layers.38.mlp.experts.linear_fc2.weight13 - module.decoder.layers.38.mlp.experts.linear_fc2.weight11 - module.decoder.layers.38.mlp.experts.linear_fc2.weight15 - module.decoder.layers.38.mlp.experts.linear_fc1.weight0 - module.decoder.layers.38.mlp.experts.linear_fc1.weight2 - module.decoder.layers.38.mlp.experts.linear_fc1.weight4 - module.decoder.layers.38.mlp.experts.linear_fc1.weight6 - module.decoder.layers.38.mlp.experts.linear_fc1.weight8 Params for bucket 19 (40894464 elements, 40894464 padded size): - module.decoder.layers.38.mlp.experts.linear_fc1.weight11 - module.decoder.layers.37.mlp.experts.linear_fc2.weight1 - module.decoder.layers.37.mlp.experts.linear_fc2.weight9 - module.decoder.layers.37.mlp.experts.linear_fc2.weight6 - module.decoder.layers.38.mlp.experts.linear_fc1.weight13 - module.decoder.layers.37.mlp.experts.linear_fc2.weight3 - module.decoder.layers.37.mlp.experts.linear_fc2.weight11 - module.decoder.layers.38.mlp.experts.linear_fc1.weight10 module.decoder.layers.37.mlp.experts.linear_fc2.weight0 module.decoder.layers.37.mlp.experts.linear_fc2.weight8 module.decoder.layers.38.mlp.experts.linear_fc1.weight15 @@ -1353,25 +1345,39 @@ Params for bucket 19 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc2.weight2 module.decoder.layers.37.mlp.experts.linear_fc2.weight10 module.decoder.layers.37.mlp.experts.linear_fc2.weight7 + module.decoder.layers.38.mlp.experts.linear_fc1.weight10 module.decoder.layers.38.mlp.experts.linear_fc1.weight14 module.decoder.layers.37.mlp.experts.linear_fc2.weight4 module.decoder.layers.37.mlp.experts.linear_fc2.weight12 + module.decoder.layers.38.mlp.experts.linear_fc1.weight11 + module.decoder.layers.37.mlp.experts.linear_fc2.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight9 + module.decoder.layers.37.mlp.experts.linear_fc2.weight6 + module.decoder.layers.38.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.37.mlp.experts.linear_fc2.weight11 Params for bucket 20 (40894464 elements, 40894464 padded size): - module.decoder.layers.37.mlp.experts.linear_fc2.weight15 - module.decoder.layers.37.mlp.experts.linear_fc1.weight0 + module.decoder.layers.37.mlp.experts.linear_fc2.weight14 module.decoder.layers.37.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc1.weight0 module.decoder.layers.37.mlp.experts.linear_fc1.weight4 module.decoder.layers.37.mlp.experts.linear_fc1.weight6 module.decoder.layers.37.mlp.experts.linear_fc1.weight8 module.decoder.layers.37.mlp.experts.linear_fc1.weight10 - module.decoder.layers.37.mlp.experts.linear_fc2.weight14 module.decoder.layers.37.mlp.experts.linear_fc1.weight1 + module.decoder.layers.37.mlp.experts.linear_fc2.weight15 module.decoder.layers.37.mlp.experts.linear_fc1.weight3 module.decoder.layers.37.mlp.experts.linear_fc1.weight5 module.decoder.layers.37.mlp.experts.linear_fc1.weight7 module.decoder.layers.37.mlp.experts.linear_fc1.weight9 module.decoder.layers.37.mlp.experts.linear_fc1.weight11 Params for bucket 21 (40894464 elements, 40894464 padded size): + module.decoder.layers.36.mlp.experts.linear_fc2.weight0 + module.decoder.layers.36.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 + module.decoder.layers.36.mlp.experts.linear_fc2.weight5 + module.decoder.layers.36.mlp.experts.linear_fc2.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight0 module.decoder.layers.37.mlp.experts.linear_fc1.weight12 module.decoder.layers.36.mlp.experts.linear_fc2.weight2 module.decoder.layers.36.mlp.experts.linear_fc2.weight10 @@ -1387,12 +1393,6 @@ Params for bucket 21 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc1.weight13 module.decoder.layers.36.mlp.experts.linear_fc2.weight3 module.decoder.layers.36.mlp.experts.linear_fc2.weight11 - module.decoder.layers.36.mlp.experts.linear_fc2.weight0 - module.decoder.layers.36.mlp.experts.linear_fc2.weight8 - module.decoder.layers.37.mlp.experts.linear_fc1.weight15 - module.decoder.layers.36.mlp.experts.linear_fc2.weight5 - module.decoder.layers.36.mlp.experts.linear_fc2.weight13 - module.decoder.layers.36.mlp.experts.linear_fc1.weight0 Params for bucket 22 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc1.weight1 module.decoder.layers.36.mlp.experts.linear_fc1.weight5 @@ -1408,6 +1408,11 @@ Params for bucket 22 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc1.weight10 module.decoder.layers.36.mlp.experts.linear_fc1.weight12 Params for bucket 23 (40894464 elements, 40894464 padded size): + module.decoder.layers.35.mlp.experts.linear_fc2.weight1 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 + module.decoder.layers.35.mlp.experts.linear_fc2.weight6 + module.decoder.layers.35.mlp.experts.linear_fc2.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 module.decoder.layers.35.mlp.experts.linear_fc2.weight3 module.decoder.layers.35.mlp.experts.linear_fc2.weight11 module.decoder.layers.35.mlp.experts.linear_fc2.weight0 @@ -1424,11 +1429,6 @@ Params for bucket 23 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc1.weight14 module.decoder.layers.35.mlp.experts.linear_fc2.weight4 module.decoder.layers.35.mlp.experts.linear_fc2.weight12 - module.decoder.layers.35.mlp.experts.linear_fc2.weight1 - module.decoder.layers.35.mlp.experts.linear_fc2.weight9 - module.decoder.layers.35.mlp.experts.linear_fc2.weight6 - module.decoder.layers.35.mlp.experts.linear_fc2.weight14 - module.decoder.layers.35.mlp.experts.linear_fc1.weight1 Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight4 module.decoder.layers.35.mlp.experts.linear_fc1.weight6 @@ -1444,16 +1444,6 @@ Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight13 module.decoder.layers.35.mlp.experts.linear_fc1.weight15 Params for bucket 25 (40894464 elements, 40894464 padded size): - module.decoder.layers.34.mlp.experts.linear_fc2.weight0 - module.decoder.layers.34.mlp.experts.linear_fc2.weight8 - module.decoder.layers.34.mlp.experts.linear_fc2.weight5 - module.decoder.layers.34.mlp.experts.linear_fc2.weight13 - module.decoder.layers.34.mlp.experts.linear_fc1.weight4 - module.decoder.layers.34.mlp.experts.linear_fc2.weight2 - module.decoder.layers.34.mlp.experts.linear_fc2.weight10 - module.decoder.layers.34.mlp.experts.linear_fc1.weight1 - module.decoder.layers.34.mlp.experts.linear_fc2.weight7 - module.decoder.layers.34.mlp.experts.linear_fc2.weight15 module.decoder.layers.34.mlp.experts.linear_fc2.weight4 module.decoder.layers.34.mlp.experts.linear_fc2.weight12 module.decoder.layers.34.mlp.experts.linear_fc1.weight3 @@ -1465,10 +1455,20 @@ Params for bucket 25 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc2.weight3 module.decoder.layers.34.mlp.experts.linear_fc2.weight11 module.decoder.layers.34.mlp.experts.linear_fc1.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight0 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc1.weight4 + module.decoder.layers.34.mlp.experts.linear_fc2.weight2 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc1.weight1 + module.decoder.layers.34.mlp.experts.linear_fc2.weight7 + module.decoder.layers.34.mlp.experts.linear_fc2.weight15 Params for bucket 26 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc1.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight7 module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.34.mlp.experts.linear_fc1.weight7 module.decoder.layers.34.mlp.experts.linear_fc1.weight11 module.decoder.layers.34.mlp.experts.linear_fc1.weight13 module.decoder.layers.34.mlp.experts.linear_fc1.weight15 @@ -1482,14 +1482,6 @@ Params for bucket 26 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc2.weight0 module.decoder.layers.33.mlp.experts.linear_fc2.weight2 Params for bucket 27 (40894464 elements, 40894464 padded size): - module.decoder.layers.33.mlp.experts.linear_fc2.weight9 - module.decoder.layers.33.mlp.experts.linear_fc1.weight0 - module.decoder.layers.33.mlp.experts.linear_fc2.weight6 - module.decoder.layers.33.mlp.experts.linear_fc2.weight14 - module.decoder.layers.33.mlp.experts.linear_fc1.weight5 - module.decoder.layers.33.mlp.experts.linear_fc2.weight11 - module.decoder.layers.33.mlp.experts.linear_fc1.weight2 - module.decoder.layers.33.mlp.experts.linear_fc2.weight8 module.decoder.layers.33.mlp.experts.linear_fc2.weight5 module.decoder.layers.33.mlp.experts.linear_fc2.weight13 module.decoder.layers.33.mlp.experts.linear_fc1.weight4 @@ -1501,6 +1493,14 @@ Params for bucket 27 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc2.weight4 module.decoder.layers.33.mlp.experts.linear_fc2.weight12 module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc2.weight6 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc1.weight5 + module.decoder.layers.33.mlp.experts.linear_fc2.weight11 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc2.weight8 Params for bucket 28 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc1.weight10 module.decoder.layers.33.mlp.experts.linear_fc1.weight8 @@ -1521,8 +1521,8 @@ Params for bucket 28 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc2.weight7 Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc2.weight8 - module.decoder.layers.32.mlp.experts.linear_fc2.weight12 module.decoder.layers.32.mlp.experts.linear_fc2.weight10 + module.decoder.layers.32.mlp.experts.linear_fc2.weight12 module.decoder.layers.32.mlp.experts.linear_fc2.weight14 module.decoder.layers.32.mlp.experts.linear_fc1.weight1 module.decoder.layers.32.mlp.experts.linear_fc1.weight3 @@ -1538,6 +1538,14 @@ Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight6 module.decoder.layers.32.mlp.experts.linear_fc1.weight8 Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight7 + module.decoder.layers.32.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc2.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 + module.decoder.layers.31.mlp.experts.linear_fc2.weight9 + module.decoder.layers.31.mlp.experts.linear_fc2.weight6 module.decoder.layers.32.mlp.experts.linear_fc1.weight13 module.decoder.layers.31.mlp.experts.linear_fc2.weight3 module.decoder.layers.31.mlp.experts.linear_fc2.weight11 @@ -1549,18 +1557,10 @@ Params for bucket 30 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight12 module.decoder.layers.31.mlp.experts.linear_fc2.weight2 module.decoder.layers.31.mlp.experts.linear_fc2.weight10 - module.decoder.layers.32.mlp.experts.linear_fc1.weight9 - module.decoder.layers.31.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc1.weight14 - module.decoder.layers.31.mlp.experts.linear_fc2.weight4 - module.decoder.layers.32.mlp.experts.linear_fc1.weight11 - module.decoder.layers.31.mlp.experts.linear_fc2.weight1 - module.decoder.layers.31.mlp.experts.linear_fc2.weight9 - module.decoder.layers.31.mlp.experts.linear_fc2.weight6 Params for bucket 31 (40894464 elements, 40894464 padded size): module.decoder.layers.31.mlp.experts.linear_fc2.weight15 - module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc1.weight2 module.decoder.layers.31.mlp.experts.linear_fc1.weight4 module.decoder.layers.31.mlp.experts.linear_fc1.weight6 @@ -1574,30 +1574,30 @@ Params for bucket 31 (40894464 elements, 40894464 padded size): module.decoder.layers.31.mlp.experts.linear_fc1.weight7 module.decoder.layers.31.mlp.experts.linear_fc1.weight9 Params for bucket 32 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.31.mlp.experts.linear_fc1.weight15 module.decoder.layers.30.mlp.experts.linear_fc2.weight4 module.decoder.layers.30.mlp.experts.linear_fc2.weight12 - module.decoder.layers.31.mlp.experts.linear_fc1.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 module.decoder.layers.30.mlp.experts.linear_fc2.weight1 module.decoder.layers.30.mlp.experts.linear_fc2.weight9 module.decoder.layers.30.mlp.experts.linear_fc2.weight6 module.decoder.layers.30.mlp.experts.linear_fc2.weight14 - module.decoder.layers.31.mlp.experts.linear_fc1.weight13 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 module.decoder.layers.30.mlp.experts.linear_fc2.weight3 module.decoder.layers.30.mlp.experts.linear_fc2.weight11 + module.decoder.layers.31.mlp.experts.linear_fc1.weight11 module.decoder.layers.30.mlp.experts.linear_fc2.weight0 module.decoder.layers.30.mlp.experts.linear_fc2.weight8 - module.decoder.layers.31.mlp.experts.linear_fc1.weight15 module.decoder.layers.30.mlp.experts.linear_fc2.weight5 module.decoder.layers.30.mlp.experts.linear_fc2.weight13 - module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight13 module.decoder.layers.30.mlp.experts.linear_fc2.weight2 module.decoder.layers.30.mlp.experts.linear_fc2.weight10 - module.decoder.layers.30.mlp.experts.linear_fc2.weight7 - module.decoder.layers.30.mlp.experts.linear_fc2.weight15 Params for bucket 33 (40894464 elements, 40894464 padded size): - module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.30.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.30.mlp.experts.linear_fc1.weight5 module.decoder.layers.30.mlp.experts.linear_fc1.weight7 module.decoder.layers.30.mlp.experts.linear_fc1.weight9 @@ -1610,16 +1610,10 @@ Params for bucket 33 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc1.weight10 module.decoder.layers.30.mlp.experts.linear_fc1.weight12 Params for bucket 34 (40894464 elements, 40894464 padded size): - module.decoder.layers.30.mlp.experts.linear_fc1.weight15 - module.decoder.layers.29.mlp.experts.linear_fc2.weight1 - module.decoder.layers.29.mlp.experts.linear_fc2.weight9 - module.decoder.layers.29.mlp.experts.linear_fc1.weight0 - module.decoder.layers.29.mlp.experts.linear_fc2.weight6 - module.decoder.layers.29.mlp.experts.linear_fc2.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight3 module.decoder.layers.29.mlp.experts.linear_fc2.weight11 - module.decoder.layers.30.mlp.experts.linear_fc1.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight8 module.decoder.layers.29.mlp.experts.linear_fc2.weight5 module.decoder.layers.29.mlp.experts.linear_fc2.weight13 @@ -1631,10 +1625,16 @@ Params for bucket 34 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc2.weight15 module.decoder.layers.29.mlp.experts.linear_fc2.weight4 module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight15 + module.decoder.layers.29.mlp.experts.linear_fc2.weight1 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 + module.decoder.layers.29.mlp.experts.linear_fc2.weight6 + module.decoder.layers.29.mlp.experts.linear_fc2.weight14 Params for bucket 35 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc1.weight2 - module.decoder.layers.29.mlp.experts.linear_fc1.weight4 module.decoder.layers.29.mlp.experts.linear_fc1.weight6 + module.decoder.layers.29.mlp.experts.linear_fc1.weight4 module.decoder.layers.29.mlp.experts.linear_fc1.weight8 module.decoder.layers.29.mlp.experts.linear_fc1.weight10 module.decoder.layers.29.mlp.experts.linear_fc1.weight12 @@ -1646,11 +1646,6 @@ Params for bucket 35 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc1.weight11 module.decoder.layers.29.mlp.experts.linear_fc1.weight13 Params for bucket 36 (40894464 elements, 40894464 padded size): - module.decoder.layers.28.mlp.experts.linear_fc2.weight2 - module.decoder.layers.28.mlp.experts.linear_fc2.weight10 - module.decoder.layers.28.mlp.experts.linear_fc1.weight1 - module.decoder.layers.28.mlp.experts.linear_fc2.weight7 - module.decoder.layers.28.mlp.experts.linear_fc2.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight4 module.decoder.layers.28.mlp.experts.linear_fc2.weight12 module.decoder.layers.28.mlp.experts.linear_fc1.weight3 @@ -1667,26 +1662,27 @@ Params for bucket 36 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc2.weight8 module.decoder.layers.28.mlp.experts.linear_fc2.weight5 module.decoder.layers.28.mlp.experts.linear_fc2.weight13 + module.decoder.layers.28.mlp.experts.linear_fc2.weight2 + module.decoder.layers.28.mlp.experts.linear_fc2.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight1 + module.decoder.layers.28.mlp.experts.linear_fc2.weight7 + module.decoder.layers.28.mlp.experts.linear_fc2.weight15 Params for bucket 37 (40894464 elements, 40894464 padded size): - module.decoder.layers.28.mlp.experts.linear_fc1.weight7 module.decoder.layers.28.mlp.experts.linear_fc1.weight5 + module.decoder.layers.28.mlp.experts.linear_fc1.weight7 module.decoder.layers.28.mlp.experts.linear_fc1.weight9 module.decoder.layers.28.mlp.experts.linear_fc1.weight11 module.decoder.layers.28.mlp.experts.linear_fc1.weight13 module.decoder.layers.28.mlp.experts.linear_fc1.weight15 module.decoder.layers.27.mlp.experts.linear_fc2.weight1 module.decoder.layers.28.mlp.experts.linear_fc1.weight4 - module.decoder.layers.28.mlp.experts.linear_fc1.weight6 module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 module.decoder.layers.28.mlp.experts.linear_fc1.weight10 module.decoder.layers.28.mlp.experts.linear_fc1.weight12 module.decoder.layers.28.mlp.experts.linear_fc1.weight14 module.decoder.layers.27.mlp.experts.linear_fc2.weight0 Params for bucket 38 (40894464 elements, 40894464 padded size): - module.decoder.layers.27.mlp.experts.linear_fc2.weight3 - module.decoder.layers.27.mlp.experts.linear_fc2.weight11 - module.decoder.layers.27.mlp.experts.linear_fc2.weight8 - module.decoder.layers.27.mlp.experts.linear_fc1.weight3 module.decoder.layers.27.mlp.experts.linear_fc2.weight5 module.decoder.layers.27.mlp.experts.linear_fc2.weight13 module.decoder.layers.27.mlp.experts.linear_fc1.weight0 @@ -1703,6 +1699,10 @@ Params for bucket 38 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc2.weight6 module.decoder.layers.27.mlp.experts.linear_fc2.weight14 module.decoder.layers.27.mlp.experts.linear_fc1.weight1 + module.decoder.layers.27.mlp.experts.linear_fc2.weight3 + module.decoder.layers.27.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc2.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc1.weight6 module.decoder.layers.27.mlp.experts.linear_fc1.weight8 @@ -1712,8 +1712,8 @@ Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc2.weight0 module.decoder.layers.26.mlp.experts.linear_fc2.weight2 module.decoder.layers.26.mlp.experts.linear_fc2.weight4 - module.decoder.layers.27.mlp.experts.linear_fc1.weight9 module.decoder.layers.27.mlp.experts.linear_fc1.weight7 + module.decoder.layers.27.mlp.experts.linear_fc1.weight9 module.decoder.layers.27.mlp.experts.linear_fc1.weight11 module.decoder.layers.27.mlp.experts.linear_fc1.weight13 module.decoder.layers.27.mlp.experts.linear_fc1.weight15 @@ -1722,16 +1722,16 @@ Params for bucket 39 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc2.weight5 Params for bucket 40 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc2.weight6 - module.decoder.layers.26.mlp.experts.linear_fc2.weight8 module.decoder.layers.26.mlp.experts.linear_fc2.weight10 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 module.decoder.layers.26.mlp.experts.linear_fc2.weight12 module.decoder.layers.26.mlp.experts.linear_fc2.weight14 module.decoder.layers.26.mlp.experts.linear_fc1.weight1 module.decoder.layers.26.mlp.experts.linear_fc1.weight3 module.decoder.layers.26.mlp.experts.linear_fc1.weight5 module.decoder.layers.26.mlp.experts.linear_fc1.weight7 - module.decoder.layers.26.mlp.experts.linear_fc2.weight7 module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight7 module.decoder.layers.26.mlp.experts.linear_fc2.weight11 module.decoder.layers.26.mlp.experts.linear_fc2.weight13 module.decoder.layers.26.mlp.experts.linear_fc2.weight15 @@ -1740,8 +1740,8 @@ Params for bucket 40 (40894464 elements, 40894464 padded size): module.decoder.layers.26.mlp.experts.linear_fc1.weight4 module.decoder.layers.26.mlp.experts.linear_fc1.weight6 Params for bucket 41 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc1.weight11 module.decoder.layers.26.mlp.experts.linear_fc1.weight9 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 module.decoder.layers.26.mlp.experts.linear_fc1.weight13 module.decoder.layers.26.mlp.experts.linear_fc1.weight15 module.decoder.layers.25.mlp.experts.linear_fc2.weight1 @@ -1750,8 +1750,8 @@ Params for bucket 41 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc2.weight7 module.decoder.layers.25.mlp.experts.linear_fc2.weight9 module.decoder.layers.26.mlp.experts.linear_fc1.weight8 - module.decoder.layers.26.mlp.experts.linear_fc1.weight10 module.decoder.layers.26.mlp.experts.linear_fc1.weight12 + module.decoder.layers.26.mlp.experts.linear_fc1.weight10 module.decoder.layers.26.mlp.experts.linear_fc1.weight14 module.decoder.layers.25.mlp.experts.linear_fc2.weight0 module.decoder.layers.25.mlp.experts.linear_fc2.weight2 @@ -1768,34 +1768,34 @@ Params for bucket 42 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc1.weight6 module.decoder.layers.25.mlp.experts.linear_fc1.weight8 module.decoder.layers.25.mlp.experts.linear_fc2.weight10 - module.decoder.layers.25.mlp.experts.linear_fc2.weight14 module.decoder.layers.25.mlp.experts.linear_fc2.weight12 + module.decoder.layers.25.mlp.experts.linear_fc2.weight14 module.decoder.layers.25.mlp.experts.linear_fc1.weight1 module.decoder.layers.25.mlp.experts.linear_fc1.weight3 module.decoder.layers.25.mlp.experts.linear_fc1.weight5 module.decoder.layers.25.mlp.experts.linear_fc1.weight7 module.decoder.layers.25.mlp.experts.linear_fc1.weight9 Params for bucket 43 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc2.weight10 - module.decoder.layers.25.mlp.experts.linear_fc1.weight13 - module.decoder.layers.24.mlp.experts.linear_fc2.weight7 module.decoder.layers.25.mlp.experts.linear_fc1.weight10 module.decoder.layers.24.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc2.weight4 module.decoder.layers.24.mlp.experts.linear_fc2.weight12 module.decoder.layers.25.mlp.experts.linear_fc1.weight15 module.decoder.layers.24.mlp.experts.linear_fc2.weight9 module.decoder.layers.25.mlp.experts.linear_fc1.weight12 module.decoder.layers.24.mlp.experts.linear_fc2.weight2 module.decoder.layers.24.mlp.experts.linear_fc2.weight6 - module.decoder.layers.24.mlp.experts.linear_fc2.weight3 module.decoder.layers.24.mlp.experts.linear_fc2.weight11 module.decoder.layers.25.mlp.experts.linear_fc1.weight14 + module.decoder.layers.24.mlp.experts.linear_fc2.weight4 module.decoder.layers.24.mlp.experts.linear_fc2.weight8 module.decoder.layers.25.mlp.experts.linear_fc1.weight11 module.decoder.layers.24.mlp.experts.linear_fc2.weight1 module.decoder.layers.24.mlp.experts.linear_fc2.weight5 module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight3 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 Params for bucket 44 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc2.weight14 module.decoder.layers.24.mlp.experts.linear_fc1.weight1 @@ -1804,19 +1804,14 @@ Params for bucket 44 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc1.weight7 module.decoder.layers.24.mlp.experts.linear_fc1.weight9 module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 module.decoder.layers.24.mlp.experts.linear_fc2.weight15 module.decoder.layers.24.mlp.experts.linear_fc1.weight0 - module.decoder.layers.24.mlp.experts.linear_fc1.weight2 module.decoder.layers.24.mlp.experts.linear_fc1.weight4 module.decoder.layers.24.mlp.experts.linear_fc1.weight6 module.decoder.layers.24.mlp.experts.linear_fc1.weight8 module.decoder.layers.24.mlp.experts.linear_fc1.weight10 Params for bucket 45 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc2.weight3 - module.decoder.layers.23.mlp.experts.linear_fc2.weight11 - module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc1.weight14 - module.decoder.layers.23.mlp.experts.linear_fc2.weight8 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight13 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 @@ -1833,9 +1828,14 @@ Params for bucket 45 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc2.weight6 module.decoder.layers.23.mlp.experts.linear_fc2.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight14 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc2.weight8 Params for bucket 46 (40894464 elements, 40894464 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc1.weight8 module.decoder.layers.23.mlp.experts.linear_fc1.weight10 @@ -1848,11 +1848,6 @@ Params for bucket 46 (40894464 elements, 40894464 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight11 module.decoder.layers.23.mlp.experts.linear_fc1.weight13 Params for bucket 47 (40894464 elements, 40894464 padded size): - module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.22.mlp.experts.linear_fc2.weight12 - module.decoder.layers.23.mlp.experts.linear_fc1.weight15 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc2.weight9 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight14 module.decoder.layers.22.mlp.experts.linear_fc1.weight1 @@ -1869,26 +1864,26 @@ Params for bucket 47 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight15 module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.22.mlp.experts.linear_fc2.weight12 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight9 Params for bucket 48 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 module.decoder.layers.22.mlp.experts.linear_fc1.weight9 module.decoder.layers.22.mlp.experts.linear_fc1.weight11 module.decoder.layers.22.mlp.experts.linear_fc1.weight13 module.decoder.layers.22.mlp.experts.linear_fc1.weight15 - module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight8 module.decoder.layers.22.mlp.experts.linear_fc1.weight10 module.decoder.layers.22.mlp.experts.linear_fc1.weight12 module.decoder.layers.22.mlp.experts.linear_fc1.weight14 Params for bucket 49 (40894464 elements, 40894464 padded size): - module.decoder.layers.21.mlp.experts.linear_fc2.weight6 - module.decoder.layers.21.mlp.experts.linear_fc2.weight14 - module.decoder.layers.21.mlp.experts.linear_fc1.weight1 - module.decoder.layers.21.mlp.experts.linear_fc2.weight3 - module.decoder.layers.21.mlp.experts.linear_fc2.weight11 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight3 @@ -1905,28 +1900,28 @@ Params for bucket 49 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc2.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight9 module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight14 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight11 Params for bucket 50 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight5 - module.decoder.layers.21.mlp.experts.linear_fc1.weight9 module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 module.decoder.layers.21.mlp.experts.linear_fc1.weight11 module.decoder.layers.21.mlp.experts.linear_fc1.weight13 module.decoder.layers.21.mlp.experts.linear_fc1.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc1.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight10 module.decoder.layers.21.mlp.experts.linear_fc1.weight12 module.decoder.layers.21.mlp.experts.linear_fc1.weight14 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 Params for bucket 51 (40894464 elements, 40894464 padded size): - module.decoder.layers.20.mlp.experts.linear_fc2.weight8 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 - module.decoder.layers.20.mlp.experts.linear_fc2.weight5 - module.decoder.layers.20.mlp.experts.linear_fc2.weight13 - module.decoder.layers.20.mlp.experts.linear_fc1.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight10 module.decoder.layers.20.mlp.experts.linear_fc1.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight7 @@ -1941,18 +1936,23 @@ Params for bucket 51 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.20.mlp.experts.linear_fc2.weight11 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight13 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight9 module.decoder.layers.20.mlp.experts.linear_fc1.weight11 + module.decoder.layers.20.mlp.experts.linear_fc1.weight9 module.decoder.layers.20.mlp.experts.linear_fc1.weight13 module.decoder.layers.20.mlp.experts.linear_fc1.weight15 module.decoder.layers.19.mlp.experts.linear_fc2.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight8 module.decoder.layers.20.mlp.experts.linear_fc1.weight10 + module.decoder.layers.20.mlp.experts.linear_fc1.weight8 module.decoder.layers.20.mlp.experts.linear_fc1.weight12 module.decoder.layers.20.mlp.experts.linear_fc1.weight14 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 @@ -1960,24 +1960,31 @@ Params for bucket 52 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 53 (40894464 elements, 40894464 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight11 module.decoder.layers.19.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 module.decoder.layers.19.mlp.experts.linear_fc2.weight13 module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight8 + module.decoder.layers.19.mlp.experts.linear_fc2.weight10 + module.decoder.layers.19.mlp.experts.linear_fc2.weight12 + module.decoder.layers.19.mlp.experts.linear_fc2.weight14 module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight2 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight8 - module.decoder.layers.19.mlp.experts.linear_fc2.weight8 - module.decoder.layers.19.mlp.experts.linear_fc2.weight12 - module.decoder.layers.19.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc2.weight14 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 Params for bucket 54 (40894464 elements, 40894464 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight10 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 module.decoder.layers.19.mlp.experts.linear_fc1.weight12 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight9 @@ -1990,52 +1997,51 @@ Params for bucket 54 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc1.weight13 - module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight10 - module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight15 - module.decoder.layers.18.mlp.experts.linear_fc2.weight1 - module.decoder.layers.18.mlp.experts.linear_fc2.weight9 Params for bucket 55 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight13 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.18.mlp.experts.linear_fc2.weight14 module.decoder.layers.18.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight8 module.decoder.layers.18.mlp.experts.linear_fc1.weight10 - module.decoder.layers.18.mlp.experts.linear_fc2.weight13 - module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight14 module.decoder.layers.18.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc1.weight9 Params for bucket 56 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.18.mlp.experts.linear_fc1.weight11 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight13 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight10 - module.decoder.layers.18.mlp.experts.linear_fc1.weight14 + module.decoder.layers.18.mlp.experts.linear_fc1.weight13 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight15 - module.decoder.layers.18.mlp.experts.linear_fc1.weight11 module.decoder.layers.17.mlp.experts.linear_fc2.weight4 module.decoder.layers.17.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc1.weight15 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight9 - module.decoder.layers.18.mlp.experts.linear_fc1.weight13 + module.decoder.layers.18.mlp.experts.linear_fc1.weight12 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 module.decoder.layers.17.mlp.experts.linear_fc2.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight11 - module.decoder.layers.18.mlp.experts.linear_fc1.weight15 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight8 Params for bucket 57 (40894464 elements, 40894464 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight2 @@ -2043,13 +2049,9 @@ Params for bucket 57 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight8 module.decoder.layers.17.mlp.experts.linear_fc1.weight10 module.decoder.layers.17.mlp.experts.linear_fc1.weight12 - module.decoder.layers.17.mlp.experts.linear_fc1.weight3 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc1.weight9 - module.decoder.layers.17.mlp.experts.linear_fc1.weight11 Params for bucket 58 (40894464 elements, 40894464 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 module.decoder.layers.16.mlp.experts.linear_fc2.weight14 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 @@ -2069,28 +2071,21 @@ Params for bucket 58 (40894464 elements, 40894464 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight14 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight12 - module.decoder.layers.16.mlp.experts.linear_fc2.weight1 - module.decoder.layers.16.mlp.experts.linear_fc2.weight9 Params for bucket 59 (40894464 elements, 40894464 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight7 - module.decoder.layers.16.mlp.experts.linear_fc1.weight9 - module.decoder.layers.16.mlp.experts.linear_fc1.weight11 - module.decoder.layers.16.mlp.experts.linear_fc1.weight13 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 module.decoder.layers.16.mlp.experts.linear_fc1.weight8 module.decoder.layers.16.mlp.experts.linear_fc1.weight10 module.decoder.layers.16.mlp.experts.linear_fc1.weight12 module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight9 + module.decoder.layers.16.mlp.experts.linear_fc1.weight11 + module.decoder.layers.16.mlp.experts.linear_fc1.weight13 Params for bucket 60 (40894464 elements, 40894464 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight15 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc2.weight4 - module.decoder.layers.15.mlp.experts.linear_fc2.weight12 module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc2.weight9 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 @@ -2101,38 +2096,33 @@ Params for bucket 60 (40894464 elements, 40894464 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight8 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight15 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 module.decoder.layers.15.mlp.experts.linear_fc2.weight13 module.decoder.layers.15.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight10 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 Params for bucket 61 (40894464 elements, 40894464 padded size): module.decoder.layers.15.mlp.experts.linear_fc1.weight4 - module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight10 module.decoder.layers.15.mlp.experts.linear_fc1.weight12 module.decoder.layers.15.mlp.experts.linear_fc1.weight14 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight9 module.decoder.layers.15.mlp.experts.linear_fc1.weight11 module.decoder.layers.15.mlp.experts.linear_fc1.weight13 module.decoder.layers.15.mlp.experts.linear_fc1.weight15 module.decoder.layers.14.mlp.experts.linear_fc2.weight1 Params for bucket 62 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.14.mlp.experts.linear_fc2.weight14 - module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight11 - module.decoder.layers.14.mlp.experts.linear_fc2.weight8 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight5 - module.decoder.layers.14.mlp.experts.linear_fc2.weight13 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 module.decoder.layers.14.mlp.experts.linear_fc2.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight10 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 @@ -2143,9 +2133,19 @@ Params for bucket 62 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight12 module.decoder.layers.14.mlp.experts.linear_fc2.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 Params for bucket 63 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight11 module.decoder.layers.14.mlp.experts.linear_fc1.weight13 module.decoder.layers.14.mlp.experts.linear_fc1.weight15 @@ -2171,8 +2171,8 @@ Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc2.weight8 module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 module.decoder.layers.13.mlp.experts.linear_fc2.weight12 module.decoder.layers.13.mlp.experts.linear_fc2.weight14 module.decoder.layers.13.mlp.experts.linear_fc1.weight1 @@ -2181,8 +2181,8 @@ Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight7 Params for bucket 65 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight8 - module.decoder.layers.13.mlp.experts.linear_fc1.weight12 module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 module.decoder.layers.13.mlp.experts.linear_fc1.weight14 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight2 @@ -2200,15 +2200,15 @@ Params for bucket 65 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc2.weight9 Params for bucket 66 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc2.weight10 - module.decoder.layers.12.mlp.experts.linear_fc2.weight14 module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.12.mlp.experts.linear_fc1.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight9 - module.decoder.layers.12.mlp.experts.linear_fc2.weight11 module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 module.decoder.layers.12.mlp.experts.linear_fc2.weight15 module.decoder.layers.12.mlp.experts.linear_fc1.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight2 @@ -2216,27 +2216,34 @@ Params for bucket 66 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight8 Params for bucket 67 (40894464 elements, 40894464 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight12 - module.decoder.layers.12.mlp.experts.linear_fc1.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight11 - module.decoder.layers.12.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight13 - module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight10 - module.decoder.layers.12.mlp.experts.linear_fc1.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 Params for bucket 68 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 @@ -2244,23 +2251,7 @@ Params for bucket 68 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight11 - module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 - module.decoder.layers.11.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight10 Params for bucket 69 (40894464 elements, 40894464 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight11 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc1.weight15 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 @@ -2273,13 +2264,16 @@ Params for bucket 69 (40894464 elements, 40894464 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight9 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 Params for bucket 70 (40894464 elements, 40894464 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight8 - module.decoder.layers.10.mlp.experts.linear_fc1.weight10 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 @@ -2287,8 +2281,19 @@ Params for bucket 70 (40894464 elements, 40894464 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight9 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 Params for bucket 71 (40894464 elements, 40894464 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight11 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 @@ -2296,19 +2301,14 @@ Params for bucket 71 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 Params for bucket 72 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 @@ -2317,21 +2317,13 @@ Params for bucket 72 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight12 module.decoder.layers.9.mlp.experts.linear_fc1.weight14 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 Params for bucket 73 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight12 - module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 @@ -2345,14 +2337,15 @@ Params for bucket 73 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 Params for bucket 74 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight12 - module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 @@ -2361,13 +2354,14 @@ Params for bucket 74 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 Params for bucket 75 (40894464 elements, 40894464 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 @@ -2381,7 +2375,21 @@ Params for bucket 75 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 76 (40894464 elements, 40894464 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 @@ -2391,27 +2399,19 @@ Params for bucket 76 (40894464 elements, 40894464 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 - module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 77 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight11 - module.decoder.layers.6.mlp.experts.linear_fc2.weight9 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 - module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 @@ -2438,9 +2438,9 @@ Params for bucket 78 (40894464 elements, 40894464 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight4 Params for bucket 79 (40894464 elements, 40894464 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 @@ -2455,114 +2455,121 @@ Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight9 Params for bucket 80 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight8 - module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight13 - module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight15 - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight12 - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight9 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight14 Params for bucket 81 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 module.decoder.layers.4.mlp.experts.linear_fc1.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight9 - module.decoder.layers.4.mlp.experts.linear_fc1.weight11 Params for bucket 82 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight14 - module.decoder.layers.3.mlp.experts.linear_fc2.weight8 - module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.3.mlp.experts.linear_fc2.weight13 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 - module.decoder.layers.3.mlp.experts.linear_fc2.weight10 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 83 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc1.weight9 - module.decoder.layers.3.mlp.experts.linear_fc1.weight11 - module.decoder.layers.3.mlp.experts.linear_fc1.weight13 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight8 module.decoder.layers.3.mlp.experts.linear_fc1.weight10 module.decoder.layers.3.mlp.experts.linear_fc1.weight12 module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 Params for bucket 84 (40894464 elements, 40894464 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 Params for bucket 85 (40894464 elements, 40894464 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight11 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight15 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight8 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc1.weight14 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 @@ -2576,14 +2583,15 @@ Params for bucket 86 (40894464 elements, 40894464 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight14 - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight11 - module.decoder.layers.1.mlp.experts.linear_fc2.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 87 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight8 module.decoder.layers.1.mlp.experts.linear_fc1.weight10 @@ -2592,45 +2600,37 @@ Params for bucket 87 (40894464 elements, 40894464 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight9 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight11 - module.decoder.layers.1.mlp.experts.linear_fc1.weight13 - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc2.weight4 Params for bucket 88 (40894464 elements, 40894464 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 - module.decoder.layers.0.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 89 (25165824 elements, 25165824 padded size): - module.decoder.layers.0.mlp.experts.linear_fc1.weight8 - module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 - module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:51:54 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 01:38:02 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -2638,7 +2638,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -2651,14 +2651,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-29 23:51:54 +[after dataloaders are built] datetime: 2026-08-30 01:38:03 done with setup ... -training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (5525.12, 5534.20) - train/valid/test-data-iterators-setup ..........: (67.13, 73.14) + model-and-optimizer-setup ......................: (5470.37, 5476.17) + train/valid/test-data-iterators-setup ..........: (65.12, 72.12) +training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-29 23:51:54 +[before the start of training step] datetime: 2026-08-30 01:38:03 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 sm_count : num_sms=32, num_sms_dedup=148 @@ -2670,36 +2670,37 @@ INFO:moonep.api:MoonEP comm buffer sizes: [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1280, 1024, 256, 640, 512, 3328, 256, 1408, 896, 1408, 128, 1152, 3712, 1024, 1792, 12544, 2816, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [105, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [123, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [18, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [1024, 128, 896, 128, 640, 0, 768, 128, 128, 256, 256, 256, 384, 128, 3584, 1152, 23424, 768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[66, 74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5248, 2176, 1792, 4224, 128, 384, 128, 128, 1280, 640, 2816, 3200, 128, 128, 4224, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [94, 80, 89, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [112, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [19, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] - [2026-08-29 23:52:17] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22173.8 | throughput per GPU (TFLOP/s/GPU): 4.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.357 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 - + [2026-08-30 01:38:25] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22257.0 | throughput per GPU (TFLOP/s/GPU): 4.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095285E+01 | loss scale: 1.0 | grad norm: 54.355 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.90 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 30.10 Number of parameters in most loaded shard in billions: 30.1034 Theoretical memory footprints: weight and optimizer=215316.59 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 66603.12646484375 | max allocated: 66603.126953125 | reserved: 68288.0 | max reserved: 68288.0 - [2026-08-29 23:52:17] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 815.1 | throughput per GPU (TFLOP/s/GPU): 109.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.503 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:18] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 644.3 | throughput per GPU (TFLOP/s/GPU): 138.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008740E+01 | loss scale: 1.0 | grad norm: 180.670 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:26] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 721.0 | throughput per GPU (TFLOP/s/GPU): 123.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.095488E+01 | loss scale: 1.0 | grad norm: 56.503 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:26] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 618.9 | throughput per GPU (TFLOP/s/GPU): 144.0 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 1.008933E+01 | loss scale: 1.0 | grad norm: 181.552 | number of skipped iterations: 0 | number of nan iterations: 0 | Capture CUDA graph for training!!! CUDA graph capture done!!! - [2026-08-29 23:52:20] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1586.4 | throughput per GPU (TFLOP/s/GPU): 56.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604942E+00 | loss scale: 1.0 | grad norm: 8.736 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:20] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 363.9 | throughput per GPU (TFLOP/s/GPU): 244.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.307531E+00 | loss scale: 1.0 | grad norm: 9.859 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:20] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 358.5 | throughput per GPU (TFLOP/s/GPU): 248.6 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.118978E+00 | loss scale: 1.0 | grad norm: 2.941 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:21] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 358.5 | throughput per GPU (TFLOP/s/GPU): 248.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.021599E+00 | loss scale: 1.0 | grad norm: 4.286 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:21] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 362.5 | throughput per GPU (TFLOP/s/GPU): 245.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.864960E+00 | loss scale: 1.0 | grad norm: 3.139 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:21] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 359.3 | throughput per GPU (TFLOP/s/GPU): 248.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.823440E+00 | loss scale: 1.0 | grad norm: 2.919 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:22] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 357.6 | throughput per GPU (TFLOP/s/GPU): 249.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.784576E+00 | loss scale: 1.0 | grad norm: 2.933 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:22] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 361.2 | throughput per GPU (TFLOP/s/GPU): 246.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.776314E+00 | loss scale: 1.0 | grad norm: 2.839 | number of skipped iterations: 0 | number of nan iterations: 0 | - [2026-08-29 23:52:23] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 362.0 | throughput per GPU (TFLOP/s/GPU): 246.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.748650E+00 | loss scale: 1.0 | grad norm: 2.751 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-29 23:52:23 -W0829 23:52:23.687000 250100 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0829 23:52:23.687000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250284 closing signal SIGTERM -W0829 23:52:23.688000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250285 closing signal SIGTERM -W0829 23:52:23.689000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250286 closing signal SIGTERM -W0829 23:52:23.690000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250287 closing signal SIGTERM -W0829 23:52:23.691000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250288 closing signal SIGTERM -W0829 23:52:23.693000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250289 closing signal SIGTERM -W0829 23:52:23.694000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250290 closing signal SIGTERM -W0829 23:52:23.694000 250100 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 250291 closing signal SIGTERM + [2026-08-30 01:38:28] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 1557.8 | throughput per GPU (TFLOP/s/GPU): 57.2 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.604801E+00 | loss scale: 1.0 | grad norm: 8.731 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:28] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 357.0 | throughput per GPU (TFLOP/s/GPU): 249.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.308386E+00 | loss scale: 1.0 | grad norm: 9.317 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:28] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 354.1 | throughput per GPU (TFLOP/s/GPU): 251.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.120133E+00 | loss scale: 1.0 | grad norm: 2.910 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:29] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 362.0 | throughput per GPU (TFLOP/s/GPU): 246.2 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.996365E+00 | loss scale: 1.0 | grad norm: 3.306 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:29] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 359.5 | throughput per GPU (TFLOP/s/GPU): 247.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.856530E+00 | loss scale: 1.0 | grad norm: 3.082 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:30] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 355.8 | throughput per GPU (TFLOP/s/GPU): 250.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.815993E+00 | loss scale: 1.0 | grad norm: 2.879 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:30] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 355.2 | throughput per GPU (TFLOP/s/GPU): 250.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.776857E+00 | loss scale: 1.0 | grad norm: 2.888 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:30] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 354.1 | throughput per GPU (TFLOP/s/GPU): 251.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.768001E+00 | loss scale: 1.0 | grad norm: 2.791 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 01:38:31] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 355.2 | throughput per GPU (TFLOP/s/GPU): 250.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.739430E+00 | loss scale: 1.0 | grad norm: 2.687 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 01:38:31 +[rank0]:[W830 01:38:32.694390538 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0830 01:38:33.442000 392738 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0830 01:38:33.443000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392922 closing signal SIGTERM +W0830 01:38:33.443000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392923 closing signal SIGTERM +W0830 01:38:33.444000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392924 closing signal SIGTERM +W0830 01:38:33.445000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392925 closing signal SIGTERM +W0830 01:38:33.446000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392926 closing signal SIGTERM +W0830 01:38:33.448000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392927 closing signal SIGTERM +W0830 01:38:33.449000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392928 closing signal SIGTERM +W0830 01:38:33.450000 392738 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 392929 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -2727,4 +2728,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 250100 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 392738 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/results.tsv b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/results.tsv index e80499898b95de31cfa6b149928a618c0c561e59..f3eff3cc3030d58019e3fb2ef59894d175479a21 100644 --- a/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/results.tsv +++ b/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/results.tsv @@ -1,13 +1,7 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir -QWEN3_235B moonep matched 16 8192 8 none none 1 OK 321.60 320.40 120303 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 -QWEN3_235B moonep matched 16 8192 8 none none 2 OK 322.05 320.40 120303 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2 -QWEN3_235B moonep matched 16 8192 8 none none 3 OK 322.10 320.40 120303 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3 -QWEN3_30B moonep matched 16 4096 8 none none 1 OK 361.53 358.70 116703 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1 -QWEN3_30B moonep matched 16 4096 8 none none 2 OK 361.17 358.70 116703 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2 -QWEN3_30B moonep matched 16 4096 8 none none 3 OK 360.76 357.60 116703 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3 -DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 596.03 595.70 133311 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep1 -DSV2_LITE_L24 moonep matched 8 8192 8 none none 2 OK 596.18 594.10 133311 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep2 -DSV2_LITE_L24 moonep matched 8 8192 8 none none 3 OK 595.97 594.00 133311 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/rep3 -QWEN2_57B moonep matched 8 4096 8 none none 1 OK 183.70 182.60 96737 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep1 -QWEN2_57B moonep matched 8 4096 8 none none 2 OK 184.22 182.60 96737 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep2 -QWEN2_57B moonep matched 8 4096 8 none none 3 OK 184.20 182.60 96737 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/rep3 +QWEN3_235B moonep matched 16 8192 8 none none 1 OK 317.23 316.40 120309 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 +QWEN3_235B moonep matched 16 8192 8 none none 2 OK 317.53 316.40 120309 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep2 +QWEN3_235B moonep matched 16 8192 8 none none 3 OK 317.80 316.40 120309 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep3 +QWEN3_30B moonep matched 16 4096 8 none none 1 OK 354.41 352.30 116707 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1 +QWEN3_30B moonep matched 16 4096 8 none none 2 OK 354.46 352.20 116707 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep2 +QWEN3_30B moonep matched 16 4096 8 none none 3 OK 355.16 352.20 116707 /workspace/tierb-fixed-moonep-full/moonep/tierb/tierb_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep3