AbhishekGhosh1998 commited on
Commit
d22deb8
·
verified ·
1 Parent(s): c6519f8

granularity results granularity_next_b200/tiera_topk_trace @ 2026-09-21T05:42:27Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
Files changed (13) hide show
  1. .gitattributes +2 -0
  2. granularity_next_b200/tiera_topk_trace/hf_watcher.log +3 -0
  3. granularity_next_b200/tiera_topk_trace/study.log +21 -0
  4. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep +2 -2
  5. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/train.log +0 -0
  6. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep +3 -0
  7. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/train.log +300 -280
  8. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep +3 -0
  9. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/train.log +372 -259
  10. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/verify_trace_ranks.log +6 -0
  11. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/run.env +102 -0
  12. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/train.log +0 -0
  13. granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/trace_results.tsv +3 -0
.gitattributes CHANGED
@@ -94,3 +94,5 @@ granularity_next_b200/tiera_topk_trace/echo/tierbtrace/tierbtrace_gbs16/G4_K16__
94
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625/nsys/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625.nsys-rep filter=lfs diff=lfs merge=lfs -text
95
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875/nsys/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875.nsys-rep filter=lfs diff=lfs merge=lfs -text
96
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
 
94
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625/nsys/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625.nsys-rep filter=lfs diff=lfs merge=lfs -text
95
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875/nsys/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875.nsys-rep filter=lfs diff=lfs merge=lfs -text
96
  granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep filter=lfs diff=lfs merge=lfs -text
97
+ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep filter=lfs diff=lfs merge=lfs -text
98
+ granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep filter=lfs diff=lfs merge=lfs -text
granularity_next_b200/tiera_topk_trace/hf_watcher.log CHANGED
@@ -39,3 +39,6 @@ Watching /workspace/granularity_next_b200/tiera_topk_trace -> HF every 600s (kil
39
  uploaded: /workspace/granularity_next_b200/tiera_topk_trace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_topk_trace
40
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
41
  [2026-09-21T05:32:24+00:00] backup pass
 
 
 
 
39
  uploaded: /workspace/granularity_next_b200/tiera_topk_trace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_topk_trace
40
  https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
41
  [2026-09-21T05:32:24+00:00] backup pass
42
+ uploaded: /workspace/granularity_next_b200/tiera_topk_trace -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/granularity_next_b200/tiera_topk_trace
43
+ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
44
+ [2026-09-21T05:42:27+00:00] backup pass
granularity_next_b200/tiera_topk_trace/study.log CHANGED
@@ -165,3 +165,24 @@ driver_repo: /workspace/blackwell_run_scripts @ b8bd100
165
  [guard] tearing down process group (outcome=STALLED)
166
  [05:32:23] trace verified -> G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875.nsys-rep (12M, all 8 GPUs + ultraep markers)
167
  [05:32:23] TRACE-ONLY G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375 (ranks: 0 1 2 3 4 5 6 7)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165
  [guard] tearing down process group (outcome=STALLED)
166
  [05:32:23] trace verified -> G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875.nsys-rep (12M, all 8 GPUs + ultraep markers)
167
  [05:32:23] TRACE-ONLY G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375 (ranks: 0 1 2 3 4 5 6 7)
168
+ [guard] iter 6/10 (70s)
169
+ [guard] iter 8/10 (80s)
170
+ [guard] iter 10/10 (120s)
171
+ [guard] nsys stop (flushing report)
172
+ [05:34:55] trace verified -> G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep (12M, all 8 GPUs + ultraep markers)
173
+ [05:34:55] TRACE-ONLY G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625 (ranks: 0 1 2 3 4 5 6 7)
174
+ [guard] iter 6/10 (70s)
175
+ [guard] iter 8/10 (80s)
176
+ [guard] iter 10/10 (120s)
177
+ [guard] nsys stop (flushing report)
178
+ [05:37:28] trace verified -> G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep (14M, all 8 GPUs + ultraep markers)
179
+ [05:37:28] TRACE-ONLY G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875 (ranks: 0 1 2 3 4 5 6 7)
180
+ [guard] iter 6/10 (70s)
181
+ [guard] iter 8/10 (80s)
182
+ [guard] no progress for 60s at iter 8/10 (past profile window); treating as done
183
+ [guard] nsys stop (flushing report)
184
+ [guard] tearing down process group (outcome=STALLED)
185
+ [05:40:36] trace verified -> G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep (14M, all 8 GPUs + ultraep markers)
186
+ [05:40:36] TRACE-ONLY G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667 (ranks: 0 1 2 3 4 5 6 7)
187
+ [guard] iter 6/10 (70s)
188
+ [guard] iter 8/10 (80s)
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ccb36de5c1df5199ea5ef7350c20642dfcc3518f1cf7ce829c2b7a31d96b14b2
3
- size 12191295
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b82d5f22c8f99269eaf8b9eb960d4be77b7c1754faae61f1b136be03300d6f40
3
+ size 12291016
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2fb19ecf430381c4ee52789dc07fd1290d1cc82e6371efb151b7004b5907c898
3
+ size 14557631
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/train.log CHANGED
@@ -6,10 +6,10 @@
6
  memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/memory_snapshots/snapshot.pickle
7
  repo=/workspace/Megatron-UltraEP @ 03d57ef7
8
  ==============================================================
9
- W0917 03:36:02.654000 981774 torch/distributed/run.py:766]
10
- W0917 03:36:02.654000 981774 torch/distributed/run.py:766] *****************************************
11
- W0917 03:36:02.654000 981774 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
- W0917 03:36:02.654000 981774 torch/distributed/run.py:766] *****************************************
13
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
14
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
15
  using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
@@ -17,6 +17,8 @@ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer
17
  Number of virtual stages per pipeline stage: None
18
  accumulate and all-reduce gradients in fp32 for bfloat16 data type.
19
  using torch.bfloat16 for parameters ...
 
 
20
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
21
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
22
  ------------------------ arguments ------------------------
@@ -563,10 +565,6 @@ using torch.bfloat16 for parameters ...
563
  -------------------- end of arguments ---------------------
564
  INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2
565
  > building GPT2BPETokenizer tokenizer ...
566
- /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
567
- warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
568
- /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
569
- warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
570
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
571
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
572
  > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
@@ -579,22 +577,24 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode
579
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
580
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
581
  WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
582
- [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
583
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
- [Gloo] Rank 0[Gloo] Rank is connected to 7 peer ranks. 6Expected number of connected peer ranks is : is connected to 77 peer ranks.
585
- Expected number of connected peer ranks is : 7
586
- [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
- [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
589
  [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
  [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
591
  [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
- [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
- [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
  [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
- [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
599
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
  > initialized tensor model parallel with size 1
@@ -606,24 +606,24 @@ Expected number of connected peer ranks is : 7
606
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
607
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
- [rank4]:[W917 03:36:15.192432153 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
- [rank6]:[W917 03:36:15.192434433 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
- [rank5]:[W917 03:36:15.193158877 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
- [rank7]:[W917 03:36:15.193202598 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
- [rank1]:[W917 03:36:15.193244485 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
- [rank3]:[W917 03:36:15.193338055 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
- [rank2]:[W917 03:36:15.193573241 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
  > compiling dataset index builder ...
617
  make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
618
  make: Nothing to be done for 'default'.
619
  make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
620
- >>> done with dataset index builder. Compilation time: 0.420 seconds
621
  WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
622
  > compiling and loading fused kernels ...
623
- [rank0]:[W917 03:36:15.785998424 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
624
- >>> done with compiling and loading fused kernels. Compilation time: 3.468 seconds
625
- time to initialize megatron (seconds): 7.268
626
- [after megatron is initialized] datetime: 2026-09-17 03:36:21
627
  building GPT model ...
628
  /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
629
  warnings.warn(
@@ -667,15 +667,15 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi
667
  Params for bucket 1 (206045184 elements, 206045184 padded size):
668
  module.output_layer.weight
669
  Params for bucket 2 (42479616 elements, 42479616 padded size):
 
 
 
670
  module.decoder.layers.15.mlp.router.weight
671
  module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
672
  module.decoder.layers.15.self_attention.linear_proj.weight
673
- module.decoder.final_layernorm.weight
674
- module.decoder.layers.15.self_attention.linear_qkv.weight
675
- module.decoder.layers.15.pre_mlp_layernorm.weight
676
  Params for bucket 3 (42475520 elements, 42475520 padded size):
677
- module.decoder.layers.14.pre_mlp_layernorm.weight
678
  module.decoder.layers.14.self_attention.linear_qkv.weight
 
679
  module.decoder.layers.14.mlp.router.weight
680
  module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
681
  module.decoder.layers.14.self_attention.linear_proj.weight
@@ -683,32 +683,32 @@ Params for bucket 4 (42475520 elements, 42475520 padded size):
683
  module.decoder.layers.13.self_attention.linear_qkv.weight
684
  module.decoder.layers.13.pre_mlp_layernorm.weight
685
  module.decoder.layers.13.mlp.router.weight
686
- module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
687
  module.decoder.layers.13.self_attention.linear_proj.weight
 
688
  Params for bucket 5 (42475520 elements, 42475520 padded size):
689
- module.decoder.layers.12.pre_mlp_layernorm.weight
690
  module.decoder.layers.12.self_attention.linear_qkv.weight
 
691
  module.decoder.layers.12.mlp.router.weight
692
- module.decoder.layers.12.self_attention.linear_proj.weight
693
  module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
 
694
  Params for bucket 6 (42475520 elements, 42475520 padded size):
695
  module.decoder.layers.11.mlp.router.weight
696
- module.decoder.layers.11.self_attention.linear_proj.weight
697
  module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
 
698
  module.decoder.layers.11.self_attention.linear_qkv.weight
699
  module.decoder.layers.11.pre_mlp_layernorm.weight
700
  Params for bucket 7 (42475520 elements, 42475520 padded size):
701
  module.decoder.layers.10.mlp.router.weight
702
  module.decoder.layers.10.self_attention.linear_proj.weight
703
  module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
704
- module.decoder.layers.10.self_attention.linear_qkv.weight
705
  module.decoder.layers.10.pre_mlp_layernorm.weight
 
706
  Params for bucket 8 (42475520 elements, 42475520 padded size):
 
 
707
  module.decoder.layers.9.mlp.router.weight
708
  module.decoder.layers.9.self_attention.linear_proj.weight
709
  module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
710
- module.decoder.layers.9.self_attention.linear_qkv.weight
711
- module.decoder.layers.9.pre_mlp_layernorm.weight
712
  Params for bucket 9 (42475520 elements, 42475520 padded size):
713
  module.decoder.layers.8.self_attention.linear_qkv.weight
714
  module.decoder.layers.8.pre_mlp_layernorm.weight
@@ -716,23 +716,23 @@ Params for bucket 9 (42475520 elements, 42475520 padded size):
716
  module.decoder.layers.8.self_attention.linear_proj.weight
717
  module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
718
  Params for bucket 10 (42475520 elements, 42475520 padded size):
719
- module.decoder.layers.7.self_attention.linear_qkv.weight
720
- module.decoder.layers.7.pre_mlp_layernorm.weight
721
  module.decoder.layers.7.mlp.router.weight
722
  module.decoder.layers.7.self_attention.linear_proj.weight
723
  module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
 
 
724
  Params for bucket 11 (42475520 elements, 42475520 padded size):
725
  module.decoder.layers.6.mlp.router.weight
726
- module.decoder.layers.6.self_attention.linear_proj.weight
727
  module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
728
- module.decoder.layers.6.pre_mlp_layernorm.weight
729
  module.decoder.layers.6.self_attention.linear_qkv.weight
 
730
  Params for bucket 12 (42475520 elements, 42475520 padded size):
731
- module.decoder.layers.5.mlp.router.weight
732
- module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
733
  module.decoder.layers.5.self_attention.linear_qkv.weight
734
  module.decoder.layers.5.self_attention.linear_proj.weight
735
- module.decoder.layers.5.pre_mlp_layernorm.weight
 
736
  Params for bucket 13 (42475520 elements, 42475520 padded size):
737
  module.decoder.layers.4.mlp.router.weight
738
  module.decoder.layers.4.self_attention.linear_proj.weight
@@ -746,115 +746,115 @@ Params for bucket 14 (42475520 elements, 42475520 padded size):
746
  module.decoder.layers.3.self_attention.linear_qkv.weight
747
  module.decoder.layers.3.pre_mlp_layernorm.weight
748
  Params for bucket 15 (42475520 elements, 42475520 padded size):
749
- module.decoder.layers.2.pre_mlp_layernorm.weight
750
- module.decoder.layers.2.self_attention.linear_qkv.weight
751
  module.decoder.layers.2.mlp.router.weight
752
  module.decoder.layers.2.self_attention.linear_proj.weight
753
  module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
 
 
754
  Params for bucket 16 (42475520 elements, 42475520 padded size):
755
- module.decoder.layers.1.self_attention.linear_qkv.weight
756
  module.decoder.layers.1.pre_mlp_layernorm.weight
 
757
  module.decoder.layers.1.mlp.router.weight
758
- module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
759
  module.decoder.layers.1.self_attention.linear_proj.weight
 
760
  Params for bucket 17 (42475520 elements, 42475520 padded size):
761
  module.decoder.layers.0.mlp.router.weight
762
- module.decoder.layers.0.self_attention.linear_proj.weight
763
  module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
764
- module.decoder.layers.0.self_attention.linear_qkv.weight
765
  module.decoder.layers.0.pre_mlp_layernorm.weight
 
 
766
  Params for bucket 18 (206045184 elements, 206045184 padded size):
767
  module.embedding.word_embeddings.weight
768
  INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67
769
  Params for bucket 1 (40370176 elements, 40370176 padded size):
770
  module.decoder.layers.15.mlp.experts.linear_fc2.weight15
771
- module.decoder.layers.15.mlp.experts.linear_fc2.weight11
772
  module.decoder.layers.15.mlp.experts.linear_fc2.weight13
 
773
  module.decoder.layers.15.mlp.experts.linear_fc2.weight9
774
  module.decoder.layers.15.mlp.experts.linear_fc2.weight7
775
  module.decoder.layers.15.mlp.experts.linear_fc2.weight5
776
- module.decoder.layers.15.mlp.experts.linear_fc2.weight12
777
  module.decoder.layers.15.mlp.experts.linear_fc2.weight14
 
778
  module.decoder.layers.15.mlp.experts.linear_fc2.weight10
779
  module.decoder.layers.15.mlp.experts.linear_fc2.weight8
780
  module.decoder.layers.15.mlp.experts.linear_fc2.weight6
781
  Params for bucket 2 (40370176 elements, 40370176 padded size):
782
  module.decoder.layers.15.mlp.experts.linear_fc2.weight3
783
  module.decoder.layers.15.mlp.experts.linear_fc2.weight1
784
- module.decoder.layers.15.mlp.experts.linear_fc1.weight14
785
- module.decoder.layers.15.mlp.experts.linear_fc2.weight4
786
- module.decoder.layers.15.mlp.experts.linear_fc2.weight2
787
- module.decoder.layers.15.mlp.experts.linear_fc2.weight0
788
  module.decoder.layers.15.mlp.experts.linear_fc1.weight15
789
  module.decoder.layers.15.mlp.experts.linear_fc1.weight13
 
 
 
 
790
  Params for bucket 3 (44040192 elements, 44040192 padded size):
791
- module.decoder.layers.15.mlp.experts.linear_fc1.weight12
792
- module.decoder.layers.15.mlp.experts.linear_fc1.weight8
793
- module.decoder.layers.15.mlp.experts.linear_fc1.weight10
794
  module.decoder.layers.15.mlp.experts.linear_fc1.weight11
795
  module.decoder.layers.15.mlp.experts.linear_fc1.weight9
796
  module.decoder.layers.15.mlp.experts.linear_fc1.weight7
 
 
 
797
  Params for bucket 4 (44040192 elements, 44040192 padded size):
798
- module.decoder.layers.15.mlp.experts.linear_fc1.weight6
799
- module.decoder.layers.15.mlp.experts.linear_fc1.weight2
800
- module.decoder.layers.15.mlp.experts.linear_fc1.weight4
801
  module.decoder.layers.15.mlp.experts.linear_fc1.weight5
802
  module.decoder.layers.15.mlp.experts.linear_fc1.weight3
803
  module.decoder.layers.15.mlp.experts.linear_fc1.weight1
 
 
 
804
  Params for bucket 5 (40370176 elements, 40370176 padded size):
 
 
 
 
805
  module.decoder.layers.15.mlp.experts.linear_fc1.weight0
806
- module.decoder.layers.14.mlp.experts.linear_fc2.weight13
807
  module.decoder.layers.14.mlp.experts.linear_fc2.weight15
 
808
  module.decoder.layers.14.mlp.experts.linear_fc2.weight11
809
  module.decoder.layers.14.mlp.experts.linear_fc2.weight9
810
  module.decoder.layers.14.mlp.experts.linear_fc2.weight7
811
- module.decoder.layers.14.mlp.experts.linear_fc2.weight14
812
- module.decoder.layers.14.mlp.experts.linear_fc2.weight12
813
- module.decoder.layers.14.mlp.experts.linear_fc2.weight10
814
- module.decoder.layers.14.mlp.experts.linear_fc2.weight8
815
  Params for bucket 6 (40370176 elements, 40370176 padded size):
816
- module.decoder.layers.14.mlp.experts.linear_fc2.weight5
817
- module.decoder.layers.14.mlp.experts.linear_fc2.weight3
818
- module.decoder.layers.14.mlp.experts.linear_fc2.weight1
819
- module.decoder.layers.14.mlp.experts.linear_fc1.weight14
820
  module.decoder.layers.14.mlp.experts.linear_fc2.weight6
821
- module.decoder.layers.14.mlp.experts.linear_fc2.weight2
822
  module.decoder.layers.14.mlp.experts.linear_fc2.weight4
 
823
  module.decoder.layers.14.mlp.experts.linear_fc2.weight0
824
  module.decoder.layers.14.mlp.experts.linear_fc1.weight15
 
 
 
 
825
  Params for bucket 7 (44040192 elements, 44040192 padded size):
826
- module.decoder.layers.14.mlp.experts.linear_fc1.weight10
827
- module.decoder.layers.14.mlp.experts.linear_fc1.weight12
828
- module.decoder.layers.14.mlp.experts.linear_fc1.weight8
829
  module.decoder.layers.14.mlp.experts.linear_fc1.weight13
830
  module.decoder.layers.14.mlp.experts.linear_fc1.weight11
831
  module.decoder.layers.14.mlp.experts.linear_fc1.weight9
 
 
 
832
  Params for bucket 8 (44040192 elements, 44040192 padded size):
833
- module.decoder.layers.14.mlp.experts.linear_fc1.weight6
834
- module.decoder.layers.14.mlp.experts.linear_fc1.weight4
835
- module.decoder.layers.14.mlp.experts.linear_fc1.weight2
836
  module.decoder.layers.14.mlp.experts.linear_fc1.weight7
837
  module.decoder.layers.14.mlp.experts.linear_fc1.weight3
838
  module.decoder.layers.14.mlp.experts.linear_fc1.weight5
 
 
 
839
  Params for bucket 9 (40370176 elements, 40370176 padded size):
840
- module.decoder.layers.14.mlp.experts.linear_fc1.weight0
841
  module.decoder.layers.13.mlp.experts.linear_fc2.weight14
842
  module.decoder.layers.13.mlp.experts.linear_fc2.weight12
843
  module.decoder.layers.13.mlp.experts.linear_fc2.weight10
844
- module.decoder.layers.14.mlp.experts.linear_fc1.weight1
845
  module.decoder.layers.13.mlp.experts.linear_fc2.weight15
846
  module.decoder.layers.13.mlp.experts.linear_fc2.weight13
847
  module.decoder.layers.13.mlp.experts.linear_fc2.weight11
848
  module.decoder.layers.13.mlp.experts.linear_fc2.weight9
849
  Params for bucket 10 (40370176 elements, 40370176 padded size):
850
  module.decoder.layers.13.mlp.experts.linear_fc2.weight8
851
- module.decoder.layers.13.mlp.experts.linear_fc2.weight4
852
  module.decoder.layers.13.mlp.experts.linear_fc2.weight6
 
853
  module.decoder.layers.13.mlp.experts.linear_fc2.weight2
854
  module.decoder.layers.13.mlp.experts.linear_fc2.weight0
855
  module.decoder.layers.13.mlp.experts.linear_fc1.weight15
856
- module.decoder.layers.13.mlp.experts.linear_fc2.weight7
857
  module.decoder.layers.13.mlp.experts.linear_fc2.weight5
 
858
  module.decoder.layers.13.mlp.experts.linear_fc2.weight3
859
  module.decoder.layers.13.mlp.experts.linear_fc2.weight1
860
  Params for bucket 11 (44040192 elements, 44040192 padded size):
@@ -865,8 +865,8 @@ Params for bucket 11 (44040192 elements, 44040192 padded size):
865
  module.decoder.layers.13.mlp.experts.linear_fc1.weight12
866
  module.decoder.layers.13.mlp.experts.linear_fc1.weight10
867
  Params for bucket 12 (44040192 elements, 44040192 padded size):
868
- module.decoder.layers.13.mlp.experts.linear_fc1.weight7
869
  module.decoder.layers.13.mlp.experts.linear_fc1.weight5
 
870
  module.decoder.layers.13.mlp.experts.linear_fc1.weight3
871
  module.decoder.layers.13.mlp.experts.linear_fc1.weight8
872
  module.decoder.layers.13.mlp.experts.linear_fc1.weight6
@@ -882,8 +882,8 @@ Params for bucket 13 (40370176 elements, 40370176 padded size):
882
  module.decoder.layers.12.mlp.experts.linear_fc2.weight11
883
  Params for bucket 14 (40370176 elements, 40370176 padded size):
884
  module.decoder.layers.12.mlp.experts.linear_fc2.weight10
885
- module.decoder.layers.12.mlp.experts.linear_fc2.weight6
886
  module.decoder.layers.12.mlp.experts.linear_fc2.weight8
 
887
  module.decoder.layers.12.mlp.experts.linear_fc2.weight4
888
  module.decoder.layers.12.mlp.experts.linear_fc2.weight2
889
  module.decoder.layers.12.mlp.experts.linear_fc2.weight0
@@ -896,58 +896,58 @@ Params for bucket 15 (44040192 elements, 44040192 padded size):
896
  module.decoder.layers.12.mlp.experts.linear_fc1.weight15
897
  module.decoder.layers.12.mlp.experts.linear_fc1.weight13
898
  module.decoder.layers.12.mlp.experts.linear_fc1.weight11
899
- module.decoder.layers.12.mlp.experts.linear_fc1.weight12
900
  module.decoder.layers.12.mlp.experts.linear_fc1.weight14
 
901
  module.decoder.layers.12.mlp.experts.linear_fc1.weight10
902
  Params for bucket 16 (44040192 elements, 44040192 padded size):
903
  module.decoder.layers.12.mlp.experts.linear_fc1.weight9
904
- module.decoder.layers.12.mlp.experts.linear_fc1.weight5
905
  module.decoder.layers.12.mlp.experts.linear_fc1.weight7
906
- module.decoder.layers.12.mlp.experts.linear_fc1.weight6
907
  module.decoder.layers.12.mlp.experts.linear_fc1.weight8
 
908
  module.decoder.layers.12.mlp.experts.linear_fc1.weight4
909
  Params for bucket 17 (40370176 elements, 40370176 padded size):
910
  module.decoder.layers.12.mlp.experts.linear_fc1.weight3
911
  module.decoder.layers.12.mlp.experts.linear_fc1.weight1
912
- module.decoder.layers.11.mlp.experts.linear_fc2.weight14
913
- module.decoder.layers.12.mlp.experts.linear_fc1.weight0
914
- module.decoder.layers.12.mlp.experts.linear_fc1.weight2
915
  module.decoder.layers.11.mlp.experts.linear_fc2.weight15
916
  module.decoder.layers.11.mlp.experts.linear_fc2.weight13
 
 
 
917
  Params for bucket 18 (40370176 elements, 40370176 padded size):
 
 
 
 
 
918
  module.decoder.layers.11.mlp.experts.linear_fc2.weight12
919
  module.decoder.layers.11.mlp.experts.linear_fc2.weight10
920
  module.decoder.layers.11.mlp.experts.linear_fc2.weight8
921
  module.decoder.layers.11.mlp.experts.linear_fc2.weight6
922
  module.decoder.layers.11.mlp.experts.linear_fc2.weight4
923
  module.decoder.layers.11.mlp.experts.linear_fc2.weight2
924
- module.decoder.layers.11.mlp.experts.linear_fc2.weight11
925
- module.decoder.layers.11.mlp.experts.linear_fc2.weight9
926
- module.decoder.layers.11.mlp.experts.linear_fc2.weight7
927
- module.decoder.layers.11.mlp.experts.linear_fc2.weight5
928
- module.decoder.layers.11.mlp.experts.linear_fc2.weight3
929
  Params for bucket 19 (44040192 elements, 44040192 padded size):
 
930
  module.decoder.layers.11.mlp.experts.linear_fc1.weight14
931
- module.decoder.layers.11.mlp.experts.linear_fc2.weight0
932
  module.decoder.layers.11.mlp.experts.linear_fc1.weight12
933
- module.decoder.layers.11.mlp.experts.linear_fc2.weight1
934
- module.decoder.layers.11.mlp.experts.linear_fc1.weight13
935
  module.decoder.layers.11.mlp.experts.linear_fc1.weight15
 
936
  module.decoder.layers.11.mlp.experts.linear_fc1.weight11
937
  Params for bucket 20 (44040192 elements, 44040192 padded size):
938
  module.decoder.layers.11.mlp.experts.linear_fc1.weight10
939
- module.decoder.layers.11.mlp.experts.linear_fc1.weight6
940
  module.decoder.layers.11.mlp.experts.linear_fc1.weight8
 
941
  module.decoder.layers.11.mlp.experts.linear_fc1.weight7
942
  module.decoder.layers.11.mlp.experts.linear_fc1.weight9
943
  module.decoder.layers.11.mlp.experts.linear_fc1.weight5
944
  Params for bucket 21 (40370176 elements, 40370176 padded size):
945
  module.decoder.layers.11.mlp.experts.linear_fc1.weight4
946
- module.decoder.layers.11.mlp.experts.linear_fc1.weight0
947
  module.decoder.layers.11.mlp.experts.linear_fc1.weight2
 
948
  module.decoder.layers.10.mlp.experts.linear_fc2.weight15
949
- module.decoder.layers.11.mlp.experts.linear_fc1.weight3
950
  module.decoder.layers.11.mlp.experts.linear_fc1.weight1
 
951
  Params for bucket 22 (40370176 elements, 40370176 padded size):
952
  module.decoder.layers.10.mlp.experts.linear_fc2.weight13
953
  module.decoder.layers.10.mlp.experts.linear_fc2.weight11
@@ -955,8 +955,8 @@ Params for bucket 22 (40370176 elements, 40370176 padded size):
955
  module.decoder.layers.10.mlp.experts.linear_fc2.weight7
956
  module.decoder.layers.10.mlp.experts.linear_fc2.weight5
957
  module.decoder.layers.10.mlp.experts.linear_fc2.weight14
958
- module.decoder.layers.10.mlp.experts.linear_fc2.weight10
959
  module.decoder.layers.10.mlp.experts.linear_fc2.weight12
 
960
  module.decoder.layers.10.mlp.experts.linear_fc2.weight8
961
  module.decoder.layers.10.mlp.experts.linear_fc2.weight6
962
  module.decoder.layers.10.mlp.experts.linear_fc2.weight4
@@ -965,17 +965,17 @@ Params for bucket 23 (44040192 elements, 44040192 padded size):
965
  module.decoder.layers.10.mlp.experts.linear_fc2.weight1
966
  module.decoder.layers.10.mlp.experts.linear_fc1.weight14
967
  module.decoder.layers.10.mlp.experts.linear_fc1.weight12
968
- module.decoder.layers.10.mlp.experts.linear_fc1.weight15
969
  module.decoder.layers.10.mlp.experts.linear_fc2.weight2
970
  module.decoder.layers.10.mlp.experts.linear_fc2.weight0
 
971
  module.decoder.layers.10.mlp.experts.linear_fc1.weight13
972
  Params for bucket 24 (44040192 elements, 44040192 padded size):
973
- module.decoder.layers.10.mlp.experts.linear_fc1.weight8
974
  module.decoder.layers.10.mlp.experts.linear_fc1.weight10
 
975
  module.decoder.layers.10.mlp.experts.linear_fc1.weight6
976
  module.decoder.layers.10.mlp.experts.linear_fc1.weight11
977
- module.decoder.layers.10.mlp.experts.linear_fc1.weight7
978
  module.decoder.layers.10.mlp.experts.linear_fc1.weight9
 
979
  Params for bucket 25 (44040192 elements, 44040192 padded size):
980
  module.decoder.layers.10.mlp.experts.linear_fc1.weight2
981
  module.decoder.layers.10.mlp.experts.linear_fc1.weight4
@@ -990,143 +990,143 @@ Params for bucket 26 (40370176 elements, 40370176 padded size):
990
  module.decoder.layers.9.mlp.experts.linear_fc2.weight9
991
  module.decoder.layers.9.mlp.experts.linear_fc2.weight7
992
  module.decoder.layers.9.mlp.experts.linear_fc2.weight5
993
- module.decoder.layers.9.mlp.experts.linear_fc2.weight12
994
  module.decoder.layers.9.mlp.experts.linear_fc2.weight14
 
995
  module.decoder.layers.9.mlp.experts.linear_fc2.weight10
996
  module.decoder.layers.9.mlp.experts.linear_fc2.weight8
997
  module.decoder.layers.9.mlp.experts.linear_fc2.weight6
998
  Params for bucket 27 (40370176 elements, 40370176 padded size):
999
  module.decoder.layers.9.mlp.experts.linear_fc2.weight3
1000
  module.decoder.layers.9.mlp.experts.linear_fc2.weight1
1001
- module.decoder.layers.9.mlp.experts.linear_fc1.weight14
1002
- module.decoder.layers.9.mlp.experts.linear_fc2.weight4
1003
- module.decoder.layers.9.mlp.experts.linear_fc2.weight0
1004
- module.decoder.layers.9.mlp.experts.linear_fc2.weight2
1005
  module.decoder.layers.9.mlp.experts.linear_fc1.weight15
1006
  module.decoder.layers.9.mlp.experts.linear_fc1.weight13
 
 
 
 
1007
  Params for bucket 28 (44040192 elements, 44040192 padded size):
1008
- module.decoder.layers.9.mlp.experts.linear_fc1.weight12
1009
- module.decoder.layers.9.mlp.experts.linear_fc1.weight8
1010
- module.decoder.layers.9.mlp.experts.linear_fc1.weight10
1011
  module.decoder.layers.9.mlp.experts.linear_fc1.weight11
1012
  module.decoder.layers.9.mlp.experts.linear_fc1.weight9
1013
  module.decoder.layers.9.mlp.experts.linear_fc1.weight7
 
 
 
1014
  Params for bucket 29 (44040192 elements, 44040192 padded size):
1015
- module.decoder.layers.9.mlp.experts.linear_fc1.weight6
1016
- module.decoder.layers.9.mlp.experts.linear_fc1.weight4
1017
- module.decoder.layers.9.mlp.experts.linear_fc1.weight2
1018
  module.decoder.layers.9.mlp.experts.linear_fc1.weight3
1019
  module.decoder.layers.9.mlp.experts.linear_fc1.weight5
1020
  module.decoder.layers.9.mlp.experts.linear_fc1.weight1
 
 
 
1021
  Params for bucket 30 (40370176 elements, 40370176 padded size):
 
 
 
 
1022
  module.decoder.layers.9.mlp.experts.linear_fc1.weight0
1023
- module.decoder.layers.8.mlp.experts.linear_fc2.weight13
1024
  module.decoder.layers.8.mlp.experts.linear_fc2.weight15
 
1025
  module.decoder.layers.8.mlp.experts.linear_fc2.weight11
1026
  module.decoder.layers.8.mlp.experts.linear_fc2.weight9
1027
  module.decoder.layers.8.mlp.experts.linear_fc2.weight7
1028
- module.decoder.layers.8.mlp.experts.linear_fc2.weight14
1029
- module.decoder.layers.8.mlp.experts.linear_fc2.weight12
1030
- module.decoder.layers.8.mlp.experts.linear_fc2.weight10
1031
- module.decoder.layers.8.mlp.experts.linear_fc2.weight8
1032
  Params for bucket 31 (40370176 elements, 40370176 padded size):
1033
- module.decoder.layers.8.mlp.experts.linear_fc2.weight5
1034
- module.decoder.layers.8.mlp.experts.linear_fc2.weight3
1035
- module.decoder.layers.8.mlp.experts.linear_fc2.weight1
1036
- module.decoder.layers.8.mlp.experts.linear_fc1.weight15
1037
  module.decoder.layers.8.mlp.experts.linear_fc2.weight6
1038
  module.decoder.layers.8.mlp.experts.linear_fc2.weight4
1039
  module.decoder.layers.8.mlp.experts.linear_fc2.weight2
1040
  module.decoder.layers.8.mlp.experts.linear_fc2.weight0
 
 
 
 
1041
  module.decoder.layers.8.mlp.experts.linear_fc1.weight14
1042
  Params for bucket 32 (44040192 elements, 44040192 padded size):
1043
  module.decoder.layers.8.mlp.experts.linear_fc1.weight13
1044
  module.decoder.layers.8.mlp.experts.linear_fc1.weight11
1045
  module.decoder.layers.8.mlp.experts.linear_fc1.weight9
1046
- module.decoder.layers.8.mlp.experts.linear_fc1.weight10
1047
  module.decoder.layers.8.mlp.experts.linear_fc1.weight12
 
1048
  module.decoder.layers.8.mlp.experts.linear_fc1.weight8
1049
  Params for bucket 33 (44040192 elements, 44040192 padded size):
1050
  module.decoder.layers.8.mlp.experts.linear_fc1.weight7
1051
- module.decoder.layers.8.mlp.experts.linear_fc1.weight3
1052
  module.decoder.layers.8.mlp.experts.linear_fc1.weight5
1053
- module.decoder.layers.8.mlp.experts.linear_fc1.weight4
1054
  module.decoder.layers.8.mlp.experts.linear_fc1.weight6
 
1055
  module.decoder.layers.8.mlp.experts.linear_fc1.weight2
1056
  Params for bucket 34 (40370176 elements, 40370176 padded size):
1057
  module.decoder.layers.8.mlp.experts.linear_fc1.weight1
1058
- module.decoder.layers.7.mlp.experts.linear_fc2.weight14
1059
- module.decoder.layers.7.mlp.experts.linear_fc2.weight12
1060
- module.decoder.layers.7.mlp.experts.linear_fc2.weight10
1061
  module.decoder.layers.7.mlp.experts.linear_fc2.weight15
1062
- module.decoder.layers.8.mlp.experts.linear_fc1.weight0
1063
  module.decoder.layers.7.mlp.experts.linear_fc2.weight13
1064
  module.decoder.layers.7.mlp.experts.linear_fc2.weight11
1065
  module.decoder.layers.7.mlp.experts.linear_fc2.weight9
 
 
 
 
1066
  Params for bucket 35 (40370176 elements, 40370176 padded size):
 
 
 
 
1067
  module.decoder.layers.7.mlp.experts.linear_fc2.weight8
1068
- module.decoder.layers.7.mlp.experts.linear_fc2.weight4
1069
  module.decoder.layers.7.mlp.experts.linear_fc2.weight6
 
1070
  module.decoder.layers.7.mlp.experts.linear_fc2.weight2
1071
  module.decoder.layers.7.mlp.experts.linear_fc2.weight0
1072
  module.decoder.layers.7.mlp.experts.linear_fc1.weight15
1073
- module.decoder.layers.7.mlp.experts.linear_fc2.weight5
1074
- module.decoder.layers.7.mlp.experts.linear_fc2.weight7
1075
- module.decoder.layers.7.mlp.experts.linear_fc2.weight3
1076
- module.decoder.layers.7.mlp.experts.linear_fc2.weight1
1077
  Params for bucket 36 (44040192 elements, 44040192 padded size):
1078
- module.decoder.layers.7.mlp.experts.linear_fc1.weight13
1079
- module.decoder.layers.7.mlp.experts.linear_fc1.weight11
1080
- module.decoder.layers.7.mlp.experts.linear_fc1.weight9
1081
  module.decoder.layers.7.mlp.experts.linear_fc1.weight14
1082
  module.decoder.layers.7.mlp.experts.linear_fc1.weight12
1083
  module.decoder.layers.7.mlp.experts.linear_fc1.weight10
 
 
 
1084
  Params for bucket 37 (44040192 elements, 44040192 padded size):
1085
- module.decoder.layers.7.mlp.experts.linear_fc1.weight5
1086
- module.decoder.layers.7.mlp.experts.linear_fc1.weight7
1087
- module.decoder.layers.7.mlp.experts.linear_fc1.weight3
1088
  module.decoder.layers.7.mlp.experts.linear_fc1.weight8
1089
  module.decoder.layers.7.mlp.experts.linear_fc1.weight6
1090
  module.decoder.layers.7.mlp.experts.linear_fc1.weight4
 
 
 
1091
  Params for bucket 38 (40370176 elements, 40370176 padded size):
1092
- module.decoder.layers.6.mlp.experts.linear_fc2.weight14
1093
- module.decoder.layers.7.mlp.experts.linear_fc1.weight1
1094
- module.decoder.layers.6.mlp.experts.linear_fc2.weight12
1095
  module.decoder.layers.7.mlp.experts.linear_fc1.weight2
1096
- module.decoder.layers.6.mlp.experts.linear_fc2.weight15
1097
  module.decoder.layers.7.mlp.experts.linear_fc1.weight0
 
1098
  module.decoder.layers.6.mlp.experts.linear_fc2.weight13
1099
  module.decoder.layers.6.mlp.experts.linear_fc2.weight11
 
 
 
1100
  Params for bucket 39 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1101
  module.decoder.layers.6.mlp.experts.linear_fc2.weight10
1102
  module.decoder.layers.6.mlp.experts.linear_fc2.weight6
1103
  module.decoder.layers.6.mlp.experts.linear_fc2.weight8
1104
  module.decoder.layers.6.mlp.experts.linear_fc2.weight4
1105
  module.decoder.layers.6.mlp.experts.linear_fc2.weight2
1106
  module.decoder.layers.6.mlp.experts.linear_fc2.weight0
1107
- module.decoder.layers.6.mlp.experts.linear_fc2.weight7
1108
- module.decoder.layers.6.mlp.experts.linear_fc2.weight9
1109
- module.decoder.layers.6.mlp.experts.linear_fc2.weight5
1110
- module.decoder.layers.6.mlp.experts.linear_fc2.weight3
1111
- module.decoder.layers.6.mlp.experts.linear_fc2.weight1
1112
  Params for bucket 40 (44040192 elements, 44040192 padded size):
1113
- module.decoder.layers.6.mlp.experts.linear_fc1.weight12
1114
  module.decoder.layers.6.mlp.experts.linear_fc1.weight14
 
1115
  module.decoder.layers.6.mlp.experts.linear_fc1.weight10
1116
  module.decoder.layers.6.mlp.experts.linear_fc1.weight15
1117
- module.decoder.layers.6.mlp.experts.linear_fc1.weight13
1118
  module.decoder.layers.6.mlp.experts.linear_fc1.weight11
 
1119
  Params for bucket 41 (44040192 elements, 44040192 padded size):
1120
- module.decoder.layers.6.mlp.experts.linear_fc1.weight6
1121
  module.decoder.layers.6.mlp.experts.linear_fc1.weight8
 
1122
  module.decoder.layers.6.mlp.experts.linear_fc1.weight4
1123
  module.decoder.layers.6.mlp.experts.linear_fc1.weight9
1124
- module.decoder.layers.6.mlp.experts.linear_fc1.weight5
1125
  module.decoder.layers.6.mlp.experts.linear_fc1.weight7
 
1126
  Params for bucket 42 (40370176 elements, 40370176 padded size):
1127
- module.decoder.layers.6.mlp.experts.linear_fc1.weight0
1128
  module.decoder.layers.5.mlp.experts.linear_fc2.weight15
1129
  module.decoder.layers.6.mlp.experts.linear_fc1.weight2
 
1130
  module.decoder.layers.5.mlp.experts.linear_fc2.weight13
1131
  module.decoder.layers.6.mlp.experts.linear_fc1.weight3
1132
  module.decoder.layers.6.mlp.experts.linear_fc1.weight1
@@ -1138,53 +1138,53 @@ Params for bucket 43 (40370176 elements, 40370176 padded size):
1138
  module.decoder.layers.5.mlp.experts.linear_fc2.weight5
1139
  module.decoder.layers.5.mlp.experts.linear_fc2.weight3
1140
  module.decoder.layers.5.mlp.experts.linear_fc2.weight12
1141
- module.decoder.layers.5.mlp.experts.linear_fc2.weight10
1142
  module.decoder.layers.5.mlp.experts.linear_fc2.weight8
 
1143
  module.decoder.layers.5.mlp.experts.linear_fc2.weight6
1144
  module.decoder.layers.5.mlp.experts.linear_fc2.weight4
1145
  module.decoder.layers.5.mlp.experts.linear_fc2.weight2
1146
  Params for bucket 44 (44040192 elements, 44040192 padded size):
1147
  module.decoder.layers.5.mlp.experts.linear_fc2.weight1
1148
- module.decoder.layers.5.mlp.experts.linear_fc1.weight13
1149
- module.decoder.layers.5.mlp.experts.linear_fc1.weight15
1150
- module.decoder.layers.5.mlp.experts.linear_fc1.weight11
1151
  module.decoder.layers.5.mlp.experts.linear_fc1.weight14
1152
- module.decoder.layers.5.mlp.experts.linear_fc2.weight0
1153
  module.decoder.layers.5.mlp.experts.linear_fc1.weight12
 
 
 
 
1154
  Params for bucket 45 (44040192 elements, 44040192 padded size):
1155
- module.decoder.layers.5.mlp.experts.linear_fc1.weight7
1156
- module.decoder.layers.5.mlp.experts.linear_fc1.weight9
1157
- module.decoder.layers.5.mlp.experts.linear_fc1.weight5
1158
  module.decoder.layers.5.mlp.experts.linear_fc1.weight10
1159
- module.decoder.layers.5.mlp.experts.linear_fc1.weight8
1160
  module.decoder.layers.5.mlp.experts.linear_fc1.weight6
 
 
 
 
1161
  Params for bucket 46 (40370176 elements, 40370176 padded size):
1162
- module.decoder.layers.5.mlp.experts.linear_fc1.weight3
1163
- module.decoder.layers.5.mlp.experts.linear_fc1.weight1
1164
  module.decoder.layers.5.mlp.experts.linear_fc1.weight4
1165
- module.decoder.layers.5.mlp.experts.linear_fc1.weight0
1166
  module.decoder.layers.5.mlp.experts.linear_fc1.weight2
 
1167
  module.decoder.layers.4.mlp.experts.linear_fc2.weight15
 
 
1168
  Params for bucket 47 (40370176 elements, 40370176 padded size):
1169
- module.decoder.layers.4.mlp.experts.linear_fc2.weight14
1170
- module.decoder.layers.4.mlp.experts.linear_fc2.weight10
1171
- module.decoder.layers.4.mlp.experts.linear_fc2.weight12
1172
- module.decoder.layers.4.mlp.experts.linear_fc2.weight8
1173
- module.decoder.layers.4.mlp.experts.linear_fc2.weight6
1174
- module.decoder.layers.4.mlp.experts.linear_fc2.weight4
1175
  module.decoder.layers.4.mlp.experts.linear_fc2.weight13
1176
  module.decoder.layers.4.mlp.experts.linear_fc2.weight11
1177
  module.decoder.layers.4.mlp.experts.linear_fc2.weight9
1178
  module.decoder.layers.4.mlp.experts.linear_fc2.weight7
1179
  module.decoder.layers.4.mlp.experts.linear_fc2.weight5
 
 
 
 
 
 
1180
  Params for bucket 48 (44040192 elements, 44040192 padded size):
1181
- module.decoder.layers.4.mlp.experts.linear_fc2.weight0
1182
- module.decoder.layers.4.mlp.experts.linear_fc2.weight2
1183
  module.decoder.layers.4.mlp.experts.linear_fc1.weight14
1184
  module.decoder.layers.4.mlp.experts.linear_fc1.weight12
1185
- module.decoder.layers.4.mlp.experts.linear_fc2.weight3
1186
  module.decoder.layers.4.mlp.experts.linear_fc1.weight15
1187
- module.decoder.layers.4.mlp.experts.linear_fc2.weight1
1188
  module.decoder.layers.4.mlp.experts.linear_fc1.weight13
1189
  Params for bucket 49 (44040192 elements, 44040192 padded size):
1190
  module.decoder.layers.4.mlp.experts.linear_fc1.weight10
@@ -1237,117 +1237,117 @@ Params for bucket 54 (44040192 elements, 44040192 padded size):
1237
  module.decoder.layers.3.mlp.experts.linear_fc1.weight1
1238
  Params for bucket 55 (40370176 elements, 40370176 padded size):
1239
  module.decoder.layers.3.mlp.experts.linear_fc1.weight0
1240
- module.decoder.layers.2.mlp.experts.linear_fc2.weight14
1241
- module.decoder.layers.2.mlp.experts.linear_fc2.weight12
1242
- module.decoder.layers.2.mlp.experts.linear_fc2.weight10
1243
- module.decoder.layers.2.mlp.experts.linear_fc2.weight8
1244
  module.decoder.layers.2.mlp.experts.linear_fc2.weight15
1245
  module.decoder.layers.2.mlp.experts.linear_fc2.weight13
1246
  module.decoder.layers.2.mlp.experts.linear_fc2.weight11
1247
  module.decoder.layers.2.mlp.experts.linear_fc2.weight9
1248
  module.decoder.layers.2.mlp.experts.linear_fc2.weight7
 
 
 
 
1249
  Params for bucket 56 (40370176 elements, 40370176 padded size):
1250
- module.decoder.layers.2.mlp.experts.linear_fc2.weight6
1251
- module.decoder.layers.2.mlp.experts.linear_fc2.weight4
1252
- module.decoder.layers.2.mlp.experts.linear_fc2.weight2
1253
- module.decoder.layers.2.mlp.experts.linear_fc2.weight0
1254
- module.decoder.layers.2.mlp.experts.linear_fc1.weight15
1255
  module.decoder.layers.2.mlp.experts.linear_fc2.weight5
1256
  module.decoder.layers.2.mlp.experts.linear_fc2.weight3
1257
  module.decoder.layers.2.mlp.experts.linear_fc2.weight1
1258
  module.decoder.layers.2.mlp.experts.linear_fc1.weight14
 
 
 
 
 
1259
  Params for bucket 57 (44040192 elements, 44040192 padded size):
1260
- module.decoder.layers.2.mlp.experts.linear_fc1.weight13
1261
- module.decoder.layers.2.mlp.experts.linear_fc1.weight9
1262
- module.decoder.layers.2.mlp.experts.linear_fc1.weight11
1263
  module.decoder.layers.2.mlp.experts.linear_fc1.weight12
1264
  module.decoder.layers.2.mlp.experts.linear_fc1.weight10
1265
  module.decoder.layers.2.mlp.experts.linear_fc1.weight8
 
 
 
1266
  Params for bucket 58 (44040192 elements, 44040192 padded size):
1267
- module.decoder.layers.2.mlp.experts.linear_fc1.weight7
1268
- module.decoder.layers.2.mlp.experts.linear_fc1.weight5
1269
- module.decoder.layers.2.mlp.experts.linear_fc1.weight3
1270
  module.decoder.layers.2.mlp.experts.linear_fc1.weight6
1271
  module.decoder.layers.2.mlp.experts.linear_fc1.weight4
1272
  module.decoder.layers.2.mlp.experts.linear_fc1.weight2
 
 
 
1273
  Params for bucket 59 (40370176 elements, 40370176 padded size):
1274
- module.decoder.layers.2.mlp.experts.linear_fc1.weight1
1275
- module.decoder.layers.1.mlp.experts.linear_fc2.weight14
1276
- module.decoder.layers.1.mlp.experts.linear_fc2.weight12
1277
- module.decoder.layers.1.mlp.experts.linear_fc2.weight10
1278
  module.decoder.layers.1.mlp.experts.linear_fc2.weight13
1279
  module.decoder.layers.2.mlp.experts.linear_fc1.weight0
1280
  module.decoder.layers.1.mlp.experts.linear_fc2.weight15
1281
  module.decoder.layers.1.mlp.experts.linear_fc2.weight11
1282
  module.decoder.layers.1.mlp.experts.linear_fc2.weight9
 
 
 
 
1283
  Params for bucket 60 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1284
  module.decoder.layers.1.mlp.experts.linear_fc2.weight8
1285
  module.decoder.layers.1.mlp.experts.linear_fc2.weight4
1286
  module.decoder.layers.1.mlp.experts.linear_fc2.weight6
1287
  module.decoder.layers.1.mlp.experts.linear_fc2.weight2
1288
  module.decoder.layers.1.mlp.experts.linear_fc2.weight0
1289
- module.decoder.layers.1.mlp.experts.linear_fc1.weight15
1290
- module.decoder.layers.1.mlp.experts.linear_fc2.weight7
1291
- module.decoder.layers.1.mlp.experts.linear_fc2.weight5
1292
- module.decoder.layers.1.mlp.experts.linear_fc2.weight3
1293
- module.decoder.layers.1.mlp.experts.linear_fc2.weight1
1294
  Params for bucket 61 (44040192 elements, 44040192 padded size):
1295
  module.decoder.layers.1.mlp.experts.linear_fc1.weight13
1296
  module.decoder.layers.1.mlp.experts.linear_fc1.weight11
1297
  module.decoder.layers.1.mlp.experts.linear_fc1.weight9
1298
  module.decoder.layers.1.mlp.experts.linear_fc1.weight14
1299
- module.decoder.layers.1.mlp.experts.linear_fc1.weight10
1300
  module.decoder.layers.1.mlp.experts.linear_fc1.weight12
 
1301
  Params for bucket 62 (44040192 elements, 44040192 padded size):
1302
- module.decoder.layers.1.mlp.experts.linear_fc1.weight7
1303
  module.decoder.layers.1.mlp.experts.linear_fc1.weight5
 
1304
  module.decoder.layers.1.mlp.experts.linear_fc1.weight3
1305
  module.decoder.layers.1.mlp.experts.linear_fc1.weight8
1306
- module.decoder.layers.1.mlp.experts.linear_fc1.weight4
1307
  module.decoder.layers.1.mlp.experts.linear_fc1.weight6
 
1308
  Params for bucket 63 (40370176 elements, 40370176 padded size):
1309
- module.decoder.layers.1.mlp.experts.linear_fc1.weight1
1310
- module.decoder.layers.0.mlp.experts.linear_fc2.weight14
1311
- module.decoder.layers.0.mlp.experts.linear_fc2.weight12
1312
- module.decoder.layers.1.mlp.experts.linear_fc1.weight2
1313
- module.decoder.layers.1.mlp.experts.linear_fc1.weight0
1314
  module.decoder.layers.0.mlp.experts.linear_fc2.weight15
 
1315
  module.decoder.layers.0.mlp.experts.linear_fc2.weight13
1316
  module.decoder.layers.0.mlp.experts.linear_fc2.weight11
 
 
 
 
1317
  Params for bucket 64 (40370176 elements, 40370176 padded size):
1318
- module.decoder.layers.0.mlp.experts.linear_fc2.weight10
1319
- module.decoder.layers.0.mlp.experts.linear_fc2.weight6
1320
- module.decoder.layers.0.mlp.experts.linear_fc2.weight8
1321
- module.decoder.layers.0.mlp.experts.linear_fc2.weight4
1322
- module.decoder.layers.0.mlp.experts.linear_fc2.weight2
1323
- module.decoder.layers.0.mlp.experts.linear_fc2.weight0
1324
  module.decoder.layers.0.mlp.experts.linear_fc2.weight7
1325
  module.decoder.layers.0.mlp.experts.linear_fc2.weight9
1326
  module.decoder.layers.0.mlp.experts.linear_fc2.weight5
1327
  module.decoder.layers.0.mlp.experts.linear_fc2.weight3
1328
  module.decoder.layers.0.mlp.experts.linear_fc2.weight1
 
 
 
 
 
 
1329
  Params for bucket 65 (44040192 elements, 44040192 padded size):
1330
- module.decoder.layers.0.mlp.experts.linear_fc1.weight15
1331
- module.decoder.layers.0.mlp.experts.linear_fc1.weight11
1332
- module.decoder.layers.0.mlp.experts.linear_fc1.weight13
1333
  module.decoder.layers.0.mlp.experts.linear_fc1.weight14
1334
  module.decoder.layers.0.mlp.experts.linear_fc1.weight12
1335
  module.decoder.layers.0.mlp.experts.linear_fc1.weight10
 
 
 
1336
  Params for bucket 66 (44040192 elements, 44040192 padded size):
 
 
 
1337
  module.decoder.layers.0.mlp.experts.linear_fc1.weight9
1338
  module.decoder.layers.0.mlp.experts.linear_fc1.weight7
1339
  module.decoder.layers.0.mlp.experts.linear_fc1.weight5
1340
- module.decoder.layers.0.mlp.experts.linear_fc1.weight8
1341
- module.decoder.layers.0.mlp.experts.linear_fc1.weight6
1342
- module.decoder.layers.0.mlp.experts.linear_fc1.weight4
1343
  Params for bucket 67 (29360128 elements, 29360128 padded size):
1344
  module.decoder.layers.0.mlp.experts.linear_fc1.weight3
 
1345
  module.decoder.layers.0.mlp.experts.linear_fc1.weight1
1346
  module.decoder.layers.0.mlp.experts.linear_fc1.weight2
1347
- module.decoder.layers.0.mlp.experts.linear_fc1.weight0
1348
- INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x7507e4f5e750>, config_logger_dir='')
1349
  INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1350
- [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-17 03:36:24
1351
  > building train, validation, and test datasets ...
1352
  > datasets target sizes (minimum size):
1353
  train: 160
@@ -1355,7 +1355,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1355
  test: 0
1356
  INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
1357
  > building train, validation, and test datasets for GPT ...
1358
- INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x7507e5ca5e80>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
1359
  INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
1360
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
1361
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
@@ -1368,28 +1368,24 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 68684a515e11
1368
  INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy
1369
  INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
1370
  > finished creating GPT datasets ...
1371
- [after dataloaders are built] datetime: 2026-09-17 03:36:24
1372
  done with setup ...
1373
- (min, max) time across ranks (ms):
1374
- model-and-optimizer-setup ......................: (3475.55, 3485.76)
1375
- train/valid/test-data-iterators-setup ..........: (112.75, 136.75)
1376
  training ...
 
 
 
1377
  Overwriting rerun_state_machine.current_iteration from -1 to 0...
1378
- [before the start of training step] datetime: 2026-09-17 03:36:25
1379
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1380
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1381
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1382
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1383
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1384
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1385
- /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1386
- warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1387
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1388
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1389
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1390
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1391
- /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1392
- warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1393
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1394
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1395
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
@@ -1398,6 +1394,10 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1398
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1399
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1400
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
 
 
 
 
1401
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1402
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1403
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
@@ -1428,12 +1428,12 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1428
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1429
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1430
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
 
 
1431
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1432
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1433
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1434
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1435
- /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1436
- warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1437
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1438
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1439
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
@@ -1504,39 +1504,57 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1504
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1505
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1506
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1507
- [2026-09-17 03:36:56] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 31200.9 | throughput per GPU (TFLOP/s/GPU): 2.4 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102242E+01 | loss scale: 1.0 | grad norm: 49.125 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9600 | min safe capacity: 1.1719 |
1508
  Number of parameters in transformer block in billions: 23.22
1509
  Number of parameters in embedding layers in billions: 0.41
1510
  Total number of parameters in billions: 23.63
1511
  Number of parameters in most loaded shard in billions: 23.6320
1512
  Theoretical memory footprints: weight and optimizer=169029.43 MB
1513
- [Rank 0] (after 1 iterations) memory (MB) | allocated: 58105.947265625 | max allocated: 64096.150390625 | reserved: 73444.0 | max reserved: 73444.0
1514
- [2026-09-17 03:36:57] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 838.5 | throughput per GPU (TFLOP/s/GPU): 87.8 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101875E+01 | loss scale: 1.0 | grad norm: 51.015 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1515
- [2026-09-17 03:36:57] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 461.2 | throughput per GPU (TFLOP/s/GPU): 159.6 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.011638E+01 | loss scale: 1.0 | grad norm: 160.619 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1516
  Capture CUDA graph for training!!!
1517
  CUDA graph capture done!!!
1518
- [2026-09-17 03:37:00] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 3292.3 | throughput per GPU (TFLOP/s/GPU): 22.4 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.298956E+01 | loss scale: 1.0 | grad norm: 50.870 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1519
- [2026-09-17 03:37:01] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 219.2 | throughput per GPU (TFLOP/s/GPU): 335.8 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.041332E+01 | loss scale: 1.0 | grad norm: 24.501 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1520
- [2026-09-17 03:37:01] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 218.3 | throughput per GPU (TFLOP/s/GPU): 337.3 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.003693E+01 | loss scale: 1.0 | grad norm: 14.480 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1521
- [2026-09-17 03:37:04] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3180.8 | throughput per GPU (TFLOP/s/GPU): 23.1 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.525423E+00 | loss scale: 1.0 | grad norm: 10.150 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1522
- [2026-09-17 03:37:04] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 230.8 | throughput per GPU (TFLOP/s/GPU): 319.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.977603E+00 | loss scale: 1.0 | grad norm: 8.882 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1523
  Capture range started in the application.
1524
  Capture range ended in the application.
1525
- Generating '/tmp/nsys-report-2610.qdstrm'
1526
 
1527
-
 
 
 
 
 
 
 
 
 
 
 
1528
  Collection stop is not allowed in this state.
1529
- W0917 03:38:35.477000 981774 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1530
- W0917 03:38:35.477000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981887 closing signal SIGTERM
1531
- Generated:
1532
- /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep
1533
- W0917 03:38:35.481000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981888 closing signal SIGTERM
1534
- W0917 03:38:35.485000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981889 closing signal SIGTERM
1535
- W0917 03:38:35.496000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981890 closing signal SIGTERM
1536
- W0917 03:38:35.498000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981891 closing signal SIGTERM
1537
- W0917 03:38:35.505000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981892 closing signal SIGTERM
1538
- W0917 03:38:35.506000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981893 closing signal SIGTERM
1539
- W0917 03:38:35.512000 981774 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 981894 closing signal SIGTERM
 
 
 
 
 
 
 
1540
  Traceback (most recent call last):
1541
  File "/usr/local/bin/torchrun", line 8, in <module>
1542
  sys.exit(main())
@@ -1551,21 +1569,23 @@ Traceback (most recent call last):
1551
  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1552
  return launch_agent(self._config, self._entrypoint, list(args))
1553
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1554
- File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1555
- result = agent.run()
1556
- ^^^^^^^^^^^
1557
- File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1558
- result = f(*args, **kwargs)
1559
- ^^^^^^^^^^^^^^^^^^
1560
- File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1561
- result = self._invoke_run(role)
1562
- ^^^^^^^^^^^^^^^^^^^^^^
1563
- File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1564
- time.sleep(monitor_interval)
1565
- File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1566
- raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1567
- torch.distributed.elastic.multiprocessing.api.SignalException: Process 981774 got signal: 15
1568
- The target application terminated. One or more process it created re-parented.
1569
- Waiting for termination of re-parented processes.
1570
- Use the `--wait` option to modify this behavior.
1571
- There are no active sessions.
 
 
 
6
  memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/memory_snapshots/snapshot.pickle
7
  repo=/workspace/Megatron-UltraEP @ 03d57ef7
8
  ==============================================================
9
+ W0921 05:35:04.517000 64368 torch/distributed/run.py:766]
10
+ W0921 05:35:04.517000 64368 torch/distributed/run.py:766] *****************************************
11
+ W0921 05:35:04.517000 64368 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
+ W0921 05:35:04.517000 64368 torch/distributed/run.py:766] *****************************************
13
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
14
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
15
  using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
 
17
  Number of virtual stages per pipeline stage: None
18
  accumulate and all-reduce gradients in fp32 for bfloat16 data type.
19
  using torch.bfloat16 for parameters ...
20
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
21
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
22
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
23
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
24
  ------------------------ arguments ------------------------
 
565
  -------------------- end of arguments ---------------------
566
  INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2
567
  > building GPT2BPETokenizer tokenizer ...
 
 
 
 
568
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
569
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
570
  > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
 
577
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
578
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
579
  WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
580
+ /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
581
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
582
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
583
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank [Gloo] Rank 5 is connected to 71 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is :
586
+ 7
587
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
  [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
  [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
  [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
593
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
  [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
599
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
  > initialized tensor model parallel with size 1
 
606
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
607
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
608
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [rank4]:[W921 05:35:17.502890894 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
+ [rank1]:[W921 05:35:17.502936173 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank7]:[W921 05:35:17.503254109 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank3]:[W921 05:35:17.503287532 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank5]:[W921 05:35:17.503326176 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank6]:[W921 05:35:17.503420881 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank2]:[W921 05:35:17.503499928 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
  > compiling dataset index builder ...
617
  make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
618
  make: Nothing to be done for 'default'.
619
  make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
620
+ >>> done with dataset index builder. Compilation time: 0.419 seconds
621
  WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
622
  > compiling and loading fused kernels ...
623
+ [rank0]:[W921 05:35:18.092333919 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
624
+ >>> done with compiling and loading fused kernels. Compilation time: 3.931 seconds
625
+ time to initialize megatron (seconds): 9.455
626
+ [after megatron is initialized] datetime: 2026-09-21 05:35:24
627
  building GPT model ...
628
  /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
629
  warnings.warn(
 
667
  Params for bucket 1 (206045184 elements, 206045184 padded size):
668
  module.output_layer.weight
669
  Params for bucket 2 (42479616 elements, 42479616 padded size):
670
+ module.decoder.layers.15.self_attention.linear_qkv.weight
671
+ module.decoder.layers.15.pre_mlp_layernorm.weight
672
+ module.decoder.final_layernorm.weight
673
  module.decoder.layers.15.mlp.router.weight
674
  module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
675
  module.decoder.layers.15.self_attention.linear_proj.weight
 
 
 
676
  Params for bucket 3 (42475520 elements, 42475520 padded size):
 
677
  module.decoder.layers.14.self_attention.linear_qkv.weight
678
+ module.decoder.layers.14.pre_mlp_layernorm.weight
679
  module.decoder.layers.14.mlp.router.weight
680
  module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
681
  module.decoder.layers.14.self_attention.linear_proj.weight
 
683
  module.decoder.layers.13.self_attention.linear_qkv.weight
684
  module.decoder.layers.13.pre_mlp_layernorm.weight
685
  module.decoder.layers.13.mlp.router.weight
 
686
  module.decoder.layers.13.self_attention.linear_proj.weight
687
+ module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
688
  Params for bucket 5 (42475520 elements, 42475520 padded size):
 
689
  module.decoder.layers.12.self_attention.linear_qkv.weight
690
+ module.decoder.layers.12.pre_mlp_layernorm.weight
691
  module.decoder.layers.12.mlp.router.weight
 
692
  module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
693
+ module.decoder.layers.12.self_attention.linear_proj.weight
694
  Params for bucket 6 (42475520 elements, 42475520 padded size):
695
  module.decoder.layers.11.mlp.router.weight
 
696
  module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
697
+ module.decoder.layers.11.self_attention.linear_proj.weight
698
  module.decoder.layers.11.self_attention.linear_qkv.weight
699
  module.decoder.layers.11.pre_mlp_layernorm.weight
700
  Params for bucket 7 (42475520 elements, 42475520 padded size):
701
  module.decoder.layers.10.mlp.router.weight
702
  module.decoder.layers.10.self_attention.linear_proj.weight
703
  module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
 
704
  module.decoder.layers.10.pre_mlp_layernorm.weight
705
+ module.decoder.layers.10.self_attention.linear_qkv.weight
706
  Params for bucket 8 (42475520 elements, 42475520 padded size):
707
+ module.decoder.layers.9.self_attention.linear_qkv.weight
708
+ module.decoder.layers.9.pre_mlp_layernorm.weight
709
  module.decoder.layers.9.mlp.router.weight
710
  module.decoder.layers.9.self_attention.linear_proj.weight
711
  module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
 
 
712
  Params for bucket 9 (42475520 elements, 42475520 padded size):
713
  module.decoder.layers.8.self_attention.linear_qkv.weight
714
  module.decoder.layers.8.pre_mlp_layernorm.weight
 
716
  module.decoder.layers.8.self_attention.linear_proj.weight
717
  module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
718
  Params for bucket 10 (42475520 elements, 42475520 padded size):
 
 
719
  module.decoder.layers.7.mlp.router.weight
720
  module.decoder.layers.7.self_attention.linear_proj.weight
721
  module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
722
+ module.decoder.layers.7.pre_mlp_layernorm.weight
723
+ module.decoder.layers.7.self_attention.linear_qkv.weight
724
  Params for bucket 11 (42475520 elements, 42475520 padded size):
725
  module.decoder.layers.6.mlp.router.weight
 
726
  module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
727
+ module.decoder.layers.6.self_attention.linear_proj.weight
728
  module.decoder.layers.6.self_attention.linear_qkv.weight
729
+ module.decoder.layers.6.pre_mlp_layernorm.weight
730
  Params for bucket 12 (42475520 elements, 42475520 padded size):
731
+ module.decoder.layers.5.pre_mlp_layernorm.weight
 
732
  module.decoder.layers.5.self_attention.linear_qkv.weight
733
  module.decoder.layers.5.self_attention.linear_proj.weight
734
+ module.decoder.layers.5.mlp.router.weight
735
+ module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
736
  Params for bucket 13 (42475520 elements, 42475520 padded size):
737
  module.decoder.layers.4.mlp.router.weight
738
  module.decoder.layers.4.self_attention.linear_proj.weight
 
746
  module.decoder.layers.3.self_attention.linear_qkv.weight
747
  module.decoder.layers.3.pre_mlp_layernorm.weight
748
  Params for bucket 15 (42475520 elements, 42475520 padded size):
 
 
749
  module.decoder.layers.2.mlp.router.weight
750
  module.decoder.layers.2.self_attention.linear_proj.weight
751
  module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
752
+ module.decoder.layers.2.pre_mlp_layernorm.weight
753
+ module.decoder.layers.2.self_attention.linear_qkv.weight
754
  Params for bucket 16 (42475520 elements, 42475520 padded size):
 
755
  module.decoder.layers.1.pre_mlp_layernorm.weight
756
+ module.decoder.layers.1.self_attention.linear_qkv.weight
757
  module.decoder.layers.1.mlp.router.weight
 
758
  module.decoder.layers.1.self_attention.linear_proj.weight
759
+ module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
760
  Params for bucket 17 (42475520 elements, 42475520 padded size):
761
  module.decoder.layers.0.mlp.router.weight
 
762
  module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
 
763
  module.decoder.layers.0.pre_mlp_layernorm.weight
764
+ module.decoder.layers.0.self_attention.linear_proj.weight
765
+ module.decoder.layers.0.self_attention.linear_qkv.weight
766
  Params for bucket 18 (206045184 elements, 206045184 padded size):
767
  module.embedding.word_embeddings.weight
768
  INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67
769
  Params for bucket 1 (40370176 elements, 40370176 padded size):
770
  module.decoder.layers.15.mlp.experts.linear_fc2.weight15
 
771
  module.decoder.layers.15.mlp.experts.linear_fc2.weight13
772
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight11
773
  module.decoder.layers.15.mlp.experts.linear_fc2.weight9
774
  module.decoder.layers.15.mlp.experts.linear_fc2.weight7
775
  module.decoder.layers.15.mlp.experts.linear_fc2.weight5
 
776
  module.decoder.layers.15.mlp.experts.linear_fc2.weight14
777
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight12
778
  module.decoder.layers.15.mlp.experts.linear_fc2.weight10
779
  module.decoder.layers.15.mlp.experts.linear_fc2.weight8
780
  module.decoder.layers.15.mlp.experts.linear_fc2.weight6
781
  Params for bucket 2 (40370176 elements, 40370176 padded size):
782
  module.decoder.layers.15.mlp.experts.linear_fc2.weight3
783
  module.decoder.layers.15.mlp.experts.linear_fc2.weight1
 
 
 
 
784
  module.decoder.layers.15.mlp.experts.linear_fc1.weight15
785
  module.decoder.layers.15.mlp.experts.linear_fc1.weight13
786
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight4
787
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight0
788
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight2
789
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight14
790
  Params for bucket 3 (44040192 elements, 44040192 padded size):
 
 
 
791
  module.decoder.layers.15.mlp.experts.linear_fc1.weight11
792
  module.decoder.layers.15.mlp.experts.linear_fc1.weight9
793
  module.decoder.layers.15.mlp.experts.linear_fc1.weight7
794
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight12
795
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight8
796
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight10
797
  Params for bucket 4 (44040192 elements, 44040192 padded size):
 
 
 
798
  module.decoder.layers.15.mlp.experts.linear_fc1.weight5
799
  module.decoder.layers.15.mlp.experts.linear_fc1.weight3
800
  module.decoder.layers.15.mlp.experts.linear_fc1.weight1
801
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight6
802
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight2
803
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight4
804
  Params for bucket 5 (40370176 elements, 40370176 padded size):
805
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight14
806
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight12
807
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight10
808
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight8
809
  module.decoder.layers.15.mlp.experts.linear_fc1.weight0
 
810
  module.decoder.layers.14.mlp.experts.linear_fc2.weight15
811
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight13
812
  module.decoder.layers.14.mlp.experts.linear_fc2.weight11
813
  module.decoder.layers.14.mlp.experts.linear_fc2.weight9
814
  module.decoder.layers.14.mlp.experts.linear_fc2.weight7
 
 
 
 
815
  Params for bucket 6 (40370176 elements, 40370176 padded size):
 
 
 
 
816
  module.decoder.layers.14.mlp.experts.linear_fc2.weight6
 
817
  module.decoder.layers.14.mlp.experts.linear_fc2.weight4
818
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight2
819
  module.decoder.layers.14.mlp.experts.linear_fc2.weight0
820
  module.decoder.layers.14.mlp.experts.linear_fc1.weight15
821
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight5
822
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight3
823
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight1
824
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight14
825
  Params for bucket 7 (44040192 elements, 44040192 padded size):
 
 
 
826
  module.decoder.layers.14.mlp.experts.linear_fc1.weight13
827
  module.decoder.layers.14.mlp.experts.linear_fc1.weight11
828
  module.decoder.layers.14.mlp.experts.linear_fc1.weight9
829
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight10
830
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight12
831
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight8
832
  Params for bucket 8 (44040192 elements, 44040192 padded size):
 
 
 
833
  module.decoder.layers.14.mlp.experts.linear_fc1.weight7
834
  module.decoder.layers.14.mlp.experts.linear_fc1.weight3
835
  module.decoder.layers.14.mlp.experts.linear_fc1.weight5
836
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight4
837
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight6
838
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight2
839
  Params for bucket 9 (40370176 elements, 40370176 padded size):
840
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight1
841
  module.decoder.layers.13.mlp.experts.linear_fc2.weight14
842
  module.decoder.layers.13.mlp.experts.linear_fc2.weight12
843
  module.decoder.layers.13.mlp.experts.linear_fc2.weight10
844
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight0
845
  module.decoder.layers.13.mlp.experts.linear_fc2.weight15
846
  module.decoder.layers.13.mlp.experts.linear_fc2.weight13
847
  module.decoder.layers.13.mlp.experts.linear_fc2.weight11
848
  module.decoder.layers.13.mlp.experts.linear_fc2.weight9
849
  Params for bucket 10 (40370176 elements, 40370176 padded size):
850
  module.decoder.layers.13.mlp.experts.linear_fc2.weight8
 
851
  module.decoder.layers.13.mlp.experts.linear_fc2.weight6
852
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight4
853
  module.decoder.layers.13.mlp.experts.linear_fc2.weight2
854
  module.decoder.layers.13.mlp.experts.linear_fc2.weight0
855
  module.decoder.layers.13.mlp.experts.linear_fc1.weight15
 
856
  module.decoder.layers.13.mlp.experts.linear_fc2.weight5
857
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight7
858
  module.decoder.layers.13.mlp.experts.linear_fc2.weight3
859
  module.decoder.layers.13.mlp.experts.linear_fc2.weight1
860
  Params for bucket 11 (44040192 elements, 44040192 padded size):
 
865
  module.decoder.layers.13.mlp.experts.linear_fc1.weight12
866
  module.decoder.layers.13.mlp.experts.linear_fc1.weight10
867
  Params for bucket 12 (44040192 elements, 44040192 padded size):
 
868
  module.decoder.layers.13.mlp.experts.linear_fc1.weight5
869
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight7
870
  module.decoder.layers.13.mlp.experts.linear_fc1.weight3
871
  module.decoder.layers.13.mlp.experts.linear_fc1.weight8
872
  module.decoder.layers.13.mlp.experts.linear_fc1.weight6
 
882
  module.decoder.layers.12.mlp.experts.linear_fc2.weight11
883
  Params for bucket 14 (40370176 elements, 40370176 padded size):
884
  module.decoder.layers.12.mlp.experts.linear_fc2.weight10
 
885
  module.decoder.layers.12.mlp.experts.linear_fc2.weight8
886
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight6
887
  module.decoder.layers.12.mlp.experts.linear_fc2.weight4
888
  module.decoder.layers.12.mlp.experts.linear_fc2.weight2
889
  module.decoder.layers.12.mlp.experts.linear_fc2.weight0
 
896
  module.decoder.layers.12.mlp.experts.linear_fc1.weight15
897
  module.decoder.layers.12.mlp.experts.linear_fc1.weight13
898
  module.decoder.layers.12.mlp.experts.linear_fc1.weight11
 
899
  module.decoder.layers.12.mlp.experts.linear_fc1.weight14
900
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight12
901
  module.decoder.layers.12.mlp.experts.linear_fc1.weight10
902
  Params for bucket 16 (44040192 elements, 44040192 padded size):
903
  module.decoder.layers.12.mlp.experts.linear_fc1.weight9
 
904
  module.decoder.layers.12.mlp.experts.linear_fc1.weight7
905
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight5
906
  module.decoder.layers.12.mlp.experts.linear_fc1.weight8
907
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight6
908
  module.decoder.layers.12.mlp.experts.linear_fc1.weight4
909
  Params for bucket 17 (40370176 elements, 40370176 padded size):
910
  module.decoder.layers.12.mlp.experts.linear_fc1.weight3
911
  module.decoder.layers.12.mlp.experts.linear_fc1.weight1
 
 
 
912
  module.decoder.layers.11.mlp.experts.linear_fc2.weight15
913
  module.decoder.layers.11.mlp.experts.linear_fc2.weight13
914
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight2
915
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight0
916
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight14
917
  Params for bucket 18 (40370176 elements, 40370176 padded size):
918
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight11
919
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight9
920
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight7
921
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight5
922
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight3
923
  module.decoder.layers.11.mlp.experts.linear_fc2.weight12
924
  module.decoder.layers.11.mlp.experts.linear_fc2.weight10
925
  module.decoder.layers.11.mlp.experts.linear_fc2.weight8
926
  module.decoder.layers.11.mlp.experts.linear_fc2.weight6
927
  module.decoder.layers.11.mlp.experts.linear_fc2.weight4
928
  module.decoder.layers.11.mlp.experts.linear_fc2.weight2
 
 
 
 
 
929
  Params for bucket 19 (44040192 elements, 44040192 padded size):
930
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight1
931
  module.decoder.layers.11.mlp.experts.linear_fc1.weight14
 
932
  module.decoder.layers.11.mlp.experts.linear_fc1.weight12
933
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight0
 
934
  module.decoder.layers.11.mlp.experts.linear_fc1.weight15
935
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight13
936
  module.decoder.layers.11.mlp.experts.linear_fc1.weight11
937
  Params for bucket 20 (44040192 elements, 44040192 padded size):
938
  module.decoder.layers.11.mlp.experts.linear_fc1.weight10
 
939
  module.decoder.layers.11.mlp.experts.linear_fc1.weight8
940
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight6
941
  module.decoder.layers.11.mlp.experts.linear_fc1.weight7
942
  module.decoder.layers.11.mlp.experts.linear_fc1.weight9
943
  module.decoder.layers.11.mlp.experts.linear_fc1.weight5
944
  Params for bucket 21 (40370176 elements, 40370176 padded size):
945
  module.decoder.layers.11.mlp.experts.linear_fc1.weight4
 
946
  module.decoder.layers.11.mlp.experts.linear_fc1.weight2
947
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight0
948
  module.decoder.layers.10.mlp.experts.linear_fc2.weight15
 
949
  module.decoder.layers.11.mlp.experts.linear_fc1.weight1
950
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight3
951
  Params for bucket 22 (40370176 elements, 40370176 padded size):
952
  module.decoder.layers.10.mlp.experts.linear_fc2.weight13
953
  module.decoder.layers.10.mlp.experts.linear_fc2.weight11
 
955
  module.decoder.layers.10.mlp.experts.linear_fc2.weight7
956
  module.decoder.layers.10.mlp.experts.linear_fc2.weight5
957
  module.decoder.layers.10.mlp.experts.linear_fc2.weight14
 
958
  module.decoder.layers.10.mlp.experts.linear_fc2.weight12
959
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight10
960
  module.decoder.layers.10.mlp.experts.linear_fc2.weight8
961
  module.decoder.layers.10.mlp.experts.linear_fc2.weight6
962
  module.decoder.layers.10.mlp.experts.linear_fc2.weight4
 
965
  module.decoder.layers.10.mlp.experts.linear_fc2.weight1
966
  module.decoder.layers.10.mlp.experts.linear_fc1.weight14
967
  module.decoder.layers.10.mlp.experts.linear_fc1.weight12
 
968
  module.decoder.layers.10.mlp.experts.linear_fc2.weight2
969
  module.decoder.layers.10.mlp.experts.linear_fc2.weight0
970
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight15
971
  module.decoder.layers.10.mlp.experts.linear_fc1.weight13
972
  Params for bucket 24 (44040192 elements, 44040192 padded size):
 
973
  module.decoder.layers.10.mlp.experts.linear_fc1.weight10
974
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight8
975
  module.decoder.layers.10.mlp.experts.linear_fc1.weight6
976
  module.decoder.layers.10.mlp.experts.linear_fc1.weight11
 
977
  module.decoder.layers.10.mlp.experts.linear_fc1.weight9
978
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight7
979
  Params for bucket 25 (44040192 elements, 44040192 padded size):
980
  module.decoder.layers.10.mlp.experts.linear_fc1.weight2
981
  module.decoder.layers.10.mlp.experts.linear_fc1.weight4
 
990
  module.decoder.layers.9.mlp.experts.linear_fc2.weight9
991
  module.decoder.layers.9.mlp.experts.linear_fc2.weight7
992
  module.decoder.layers.9.mlp.experts.linear_fc2.weight5
 
993
  module.decoder.layers.9.mlp.experts.linear_fc2.weight14
994
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight12
995
  module.decoder.layers.9.mlp.experts.linear_fc2.weight10
996
  module.decoder.layers.9.mlp.experts.linear_fc2.weight8
997
  module.decoder.layers.9.mlp.experts.linear_fc2.weight6
998
  Params for bucket 27 (40370176 elements, 40370176 padded size):
999
  module.decoder.layers.9.mlp.experts.linear_fc2.weight3
1000
  module.decoder.layers.9.mlp.experts.linear_fc2.weight1
 
 
 
 
1001
  module.decoder.layers.9.mlp.experts.linear_fc1.weight15
1002
  module.decoder.layers.9.mlp.experts.linear_fc1.weight13
1003
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight4
1004
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight2
1005
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight0
1006
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight14
1007
  Params for bucket 28 (44040192 elements, 44040192 padded size):
 
 
 
1008
  module.decoder.layers.9.mlp.experts.linear_fc1.weight11
1009
  module.decoder.layers.9.mlp.experts.linear_fc1.weight9
1010
  module.decoder.layers.9.mlp.experts.linear_fc1.weight7
1011
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight12
1012
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight10
1013
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight8
1014
  Params for bucket 29 (44040192 elements, 44040192 padded size):
 
 
 
1015
  module.decoder.layers.9.mlp.experts.linear_fc1.weight3
1016
  module.decoder.layers.9.mlp.experts.linear_fc1.weight5
1017
  module.decoder.layers.9.mlp.experts.linear_fc1.weight1
1018
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight6
1019
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight4
1020
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight2
1021
  Params for bucket 30 (40370176 elements, 40370176 padded size):
1022
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight14
1023
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight12
1024
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight10
1025
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight8
1026
  module.decoder.layers.9.mlp.experts.linear_fc1.weight0
 
1027
  module.decoder.layers.8.mlp.experts.linear_fc2.weight15
1028
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight13
1029
  module.decoder.layers.8.mlp.experts.linear_fc2.weight11
1030
  module.decoder.layers.8.mlp.experts.linear_fc2.weight9
1031
  module.decoder.layers.8.mlp.experts.linear_fc2.weight7
 
 
 
 
1032
  Params for bucket 31 (40370176 elements, 40370176 padded size):
 
 
 
 
1033
  module.decoder.layers.8.mlp.experts.linear_fc2.weight6
1034
  module.decoder.layers.8.mlp.experts.linear_fc2.weight4
1035
  module.decoder.layers.8.mlp.experts.linear_fc2.weight2
1036
  module.decoder.layers.8.mlp.experts.linear_fc2.weight0
1037
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight15
1038
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight5
1039
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight3
1040
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight1
1041
  module.decoder.layers.8.mlp.experts.linear_fc1.weight14
1042
  Params for bucket 32 (44040192 elements, 44040192 padded size):
1043
  module.decoder.layers.8.mlp.experts.linear_fc1.weight13
1044
  module.decoder.layers.8.mlp.experts.linear_fc1.weight11
1045
  module.decoder.layers.8.mlp.experts.linear_fc1.weight9
 
1046
  module.decoder.layers.8.mlp.experts.linear_fc1.weight12
1047
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight10
1048
  module.decoder.layers.8.mlp.experts.linear_fc1.weight8
1049
  Params for bucket 33 (44040192 elements, 44040192 padded size):
1050
  module.decoder.layers.8.mlp.experts.linear_fc1.weight7
 
1051
  module.decoder.layers.8.mlp.experts.linear_fc1.weight5
1052
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight3
1053
  module.decoder.layers.8.mlp.experts.linear_fc1.weight6
1054
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight4
1055
  module.decoder.layers.8.mlp.experts.linear_fc1.weight2
1056
  Params for bucket 34 (40370176 elements, 40370176 padded size):
1057
  module.decoder.layers.8.mlp.experts.linear_fc1.weight1
 
 
 
1058
  module.decoder.layers.7.mlp.experts.linear_fc2.weight15
 
1059
  module.decoder.layers.7.mlp.experts.linear_fc2.weight13
1060
  module.decoder.layers.7.mlp.experts.linear_fc2.weight11
1061
  module.decoder.layers.7.mlp.experts.linear_fc2.weight9
1062
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight14
1063
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight0
1064
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight12
1065
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight10
1066
  Params for bucket 35 (40370176 elements, 40370176 padded size):
1067
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight5
1068
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight7
1069
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight3
1070
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight1
1071
  module.decoder.layers.7.mlp.experts.linear_fc2.weight8
 
1072
  module.decoder.layers.7.mlp.experts.linear_fc2.weight6
1073
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight4
1074
  module.decoder.layers.7.mlp.experts.linear_fc2.weight2
1075
  module.decoder.layers.7.mlp.experts.linear_fc2.weight0
1076
  module.decoder.layers.7.mlp.experts.linear_fc1.weight15
 
 
 
 
1077
  Params for bucket 36 (44040192 elements, 44040192 padded size):
 
 
 
1078
  module.decoder.layers.7.mlp.experts.linear_fc1.weight14
1079
  module.decoder.layers.7.mlp.experts.linear_fc1.weight12
1080
  module.decoder.layers.7.mlp.experts.linear_fc1.weight10
1081
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight13
1082
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight11
1083
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight9
1084
  Params for bucket 37 (44040192 elements, 44040192 padded size):
 
 
 
1085
  module.decoder.layers.7.mlp.experts.linear_fc1.weight8
1086
  module.decoder.layers.7.mlp.experts.linear_fc1.weight6
1087
  module.decoder.layers.7.mlp.experts.linear_fc1.weight4
1088
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight7
1089
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight5
1090
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight3
1091
  Params for bucket 38 (40370176 elements, 40370176 padded size):
 
 
 
1092
  module.decoder.layers.7.mlp.experts.linear_fc1.weight2
 
1093
  module.decoder.layers.7.mlp.experts.linear_fc1.weight0
1094
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight15
1095
  module.decoder.layers.6.mlp.experts.linear_fc2.weight13
1096
  module.decoder.layers.6.mlp.experts.linear_fc2.weight11
1097
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight14
1098
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight1
1099
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight12
1100
  Params for bucket 39 (40370176 elements, 40370176 padded size):
1101
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight7
1102
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight9
1103
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight5
1104
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight3
1105
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight1
1106
  module.decoder.layers.6.mlp.experts.linear_fc2.weight10
1107
  module.decoder.layers.6.mlp.experts.linear_fc2.weight6
1108
  module.decoder.layers.6.mlp.experts.linear_fc2.weight8
1109
  module.decoder.layers.6.mlp.experts.linear_fc2.weight4
1110
  module.decoder.layers.6.mlp.experts.linear_fc2.weight2
1111
  module.decoder.layers.6.mlp.experts.linear_fc2.weight0
 
 
 
 
 
1112
  Params for bucket 40 (44040192 elements, 44040192 padded size):
 
1113
  module.decoder.layers.6.mlp.experts.linear_fc1.weight14
1114
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight12
1115
  module.decoder.layers.6.mlp.experts.linear_fc1.weight10
1116
  module.decoder.layers.6.mlp.experts.linear_fc1.weight15
 
1117
  module.decoder.layers.6.mlp.experts.linear_fc1.weight11
1118
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight13
1119
  Params for bucket 41 (44040192 elements, 44040192 padded size):
 
1120
  module.decoder.layers.6.mlp.experts.linear_fc1.weight8
1121
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight6
1122
  module.decoder.layers.6.mlp.experts.linear_fc1.weight4
1123
  module.decoder.layers.6.mlp.experts.linear_fc1.weight9
 
1124
  module.decoder.layers.6.mlp.experts.linear_fc1.weight7
1125
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight5
1126
  Params for bucket 42 (40370176 elements, 40370176 padded size):
 
1127
  module.decoder.layers.5.mlp.experts.linear_fc2.weight15
1128
  module.decoder.layers.6.mlp.experts.linear_fc1.weight2
1129
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight0
1130
  module.decoder.layers.5.mlp.experts.linear_fc2.weight13
1131
  module.decoder.layers.6.mlp.experts.linear_fc1.weight3
1132
  module.decoder.layers.6.mlp.experts.linear_fc1.weight1
 
1138
  module.decoder.layers.5.mlp.experts.linear_fc2.weight5
1139
  module.decoder.layers.5.mlp.experts.linear_fc2.weight3
1140
  module.decoder.layers.5.mlp.experts.linear_fc2.weight12
 
1141
  module.decoder.layers.5.mlp.experts.linear_fc2.weight8
1142
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight10
1143
  module.decoder.layers.5.mlp.experts.linear_fc2.weight6
1144
  module.decoder.layers.5.mlp.experts.linear_fc2.weight4
1145
  module.decoder.layers.5.mlp.experts.linear_fc2.weight2
1146
  Params for bucket 44 (44040192 elements, 44040192 padded size):
1147
  module.decoder.layers.5.mlp.experts.linear_fc2.weight1
 
 
 
1148
  module.decoder.layers.5.mlp.experts.linear_fc1.weight14
 
1149
  module.decoder.layers.5.mlp.experts.linear_fc1.weight12
1150
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight15
1151
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight13
1152
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight0
1153
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight11
1154
  Params for bucket 45 (44040192 elements, 44040192 padded size):
 
 
 
1155
  module.decoder.layers.5.mlp.experts.linear_fc1.weight10
 
1156
  module.decoder.layers.5.mlp.experts.linear_fc1.weight6
1157
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight8
1158
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight9
1159
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight7
1160
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight5
1161
  Params for bucket 46 (40370176 elements, 40370176 padded size):
 
 
1162
  module.decoder.layers.5.mlp.experts.linear_fc1.weight4
 
1163
  module.decoder.layers.5.mlp.experts.linear_fc1.weight2
1164
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight0
1165
  module.decoder.layers.4.mlp.experts.linear_fc2.weight15
1166
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight3
1167
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight1
1168
  Params for bucket 47 (40370176 elements, 40370176 padded size):
 
 
 
 
 
 
1169
  module.decoder.layers.4.mlp.experts.linear_fc2.weight13
1170
  module.decoder.layers.4.mlp.experts.linear_fc2.weight11
1171
  module.decoder.layers.4.mlp.experts.linear_fc2.weight9
1172
  module.decoder.layers.4.mlp.experts.linear_fc2.weight7
1173
  module.decoder.layers.4.mlp.experts.linear_fc2.weight5
1174
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight14
1175
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight12
1176
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight10
1177
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight8
1178
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight6
1179
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight4
1180
  Params for bucket 48 (44040192 elements, 44040192 padded size):
1181
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight3
1182
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight1
1183
  module.decoder.layers.4.mlp.experts.linear_fc1.weight14
1184
  module.decoder.layers.4.mlp.experts.linear_fc1.weight12
1185
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight2
1186
  module.decoder.layers.4.mlp.experts.linear_fc1.weight15
1187
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight0
1188
  module.decoder.layers.4.mlp.experts.linear_fc1.weight13
1189
  Params for bucket 49 (44040192 elements, 44040192 padded size):
1190
  module.decoder.layers.4.mlp.experts.linear_fc1.weight10
 
1237
  module.decoder.layers.3.mlp.experts.linear_fc1.weight1
1238
  Params for bucket 55 (40370176 elements, 40370176 padded size):
1239
  module.decoder.layers.3.mlp.experts.linear_fc1.weight0
 
 
 
 
1240
  module.decoder.layers.2.mlp.experts.linear_fc2.weight15
1241
  module.decoder.layers.2.mlp.experts.linear_fc2.weight13
1242
  module.decoder.layers.2.mlp.experts.linear_fc2.weight11
1243
  module.decoder.layers.2.mlp.experts.linear_fc2.weight9
1244
  module.decoder.layers.2.mlp.experts.linear_fc2.weight7
1245
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight12
1246
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight14
1247
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight10
1248
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight8
1249
  Params for bucket 56 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1250
  module.decoder.layers.2.mlp.experts.linear_fc2.weight5
1251
  module.decoder.layers.2.mlp.experts.linear_fc2.weight3
1252
  module.decoder.layers.2.mlp.experts.linear_fc2.weight1
1253
  module.decoder.layers.2.mlp.experts.linear_fc1.weight14
1254
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight6
1255
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight2
1256
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight4
1257
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight0
1258
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight15
1259
  Params for bucket 57 (44040192 elements, 44040192 padded size):
 
 
 
1260
  module.decoder.layers.2.mlp.experts.linear_fc1.weight12
1261
  module.decoder.layers.2.mlp.experts.linear_fc1.weight10
1262
  module.decoder.layers.2.mlp.experts.linear_fc1.weight8
1263
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight13
1264
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight9
1265
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight11
1266
  Params for bucket 58 (44040192 elements, 44040192 padded size):
 
 
 
1267
  module.decoder.layers.2.mlp.experts.linear_fc1.weight6
1268
  module.decoder.layers.2.mlp.experts.linear_fc1.weight4
1269
  module.decoder.layers.2.mlp.experts.linear_fc1.weight2
1270
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight7
1271
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight5
1272
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight3
1273
  Params for bucket 59 (40370176 elements, 40370176 padded size):
 
 
 
 
1274
  module.decoder.layers.1.mlp.experts.linear_fc2.weight13
1275
  module.decoder.layers.2.mlp.experts.linear_fc1.weight0
1276
  module.decoder.layers.1.mlp.experts.linear_fc2.weight15
1277
  module.decoder.layers.1.mlp.experts.linear_fc2.weight11
1278
  module.decoder.layers.1.mlp.experts.linear_fc2.weight9
1279
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight1
1280
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight14
1281
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight12
1282
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight10
1283
  Params for bucket 60 (40370176 elements, 40370176 padded size):
1284
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
1285
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
1286
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
1287
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
1288
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight15
1289
  module.decoder.layers.1.mlp.experts.linear_fc2.weight8
1290
  module.decoder.layers.1.mlp.experts.linear_fc2.weight4
1291
  module.decoder.layers.1.mlp.experts.linear_fc2.weight6
1292
  module.decoder.layers.1.mlp.experts.linear_fc2.weight2
1293
  module.decoder.layers.1.mlp.experts.linear_fc2.weight0
 
 
 
 
 
1294
  Params for bucket 61 (44040192 elements, 44040192 padded size):
1295
  module.decoder.layers.1.mlp.experts.linear_fc1.weight13
1296
  module.decoder.layers.1.mlp.experts.linear_fc1.weight11
1297
  module.decoder.layers.1.mlp.experts.linear_fc1.weight9
1298
  module.decoder.layers.1.mlp.experts.linear_fc1.weight14
 
1299
  module.decoder.layers.1.mlp.experts.linear_fc1.weight12
1300
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
1301
  Params for bucket 62 (44040192 elements, 44040192 padded size):
 
1302
  module.decoder.layers.1.mlp.experts.linear_fc1.weight5
1303
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
1304
  module.decoder.layers.1.mlp.experts.linear_fc1.weight3
1305
  module.decoder.layers.1.mlp.experts.linear_fc1.weight8
 
1306
  module.decoder.layers.1.mlp.experts.linear_fc1.weight6
1307
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
1308
  Params for bucket 63 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1309
  module.decoder.layers.0.mlp.experts.linear_fc2.weight15
1310
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
1311
  module.decoder.layers.0.mlp.experts.linear_fc2.weight13
1312
  module.decoder.layers.0.mlp.experts.linear_fc2.weight11
1313
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
1314
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight14
1315
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
1316
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight12
1317
  Params for bucket 64 (40370176 elements, 40370176 padded size):
 
 
 
 
 
 
1318
  module.decoder.layers.0.mlp.experts.linear_fc2.weight7
1319
  module.decoder.layers.0.mlp.experts.linear_fc2.weight9
1320
  module.decoder.layers.0.mlp.experts.linear_fc2.weight5
1321
  module.decoder.layers.0.mlp.experts.linear_fc2.weight3
1322
  module.decoder.layers.0.mlp.experts.linear_fc2.weight1
1323
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight10
1324
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight8
1325
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight6
1326
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight4
1327
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight2
1328
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight0
1329
  Params for bucket 65 (44040192 elements, 44040192 padded size):
 
 
 
1330
  module.decoder.layers.0.mlp.experts.linear_fc1.weight14
1331
  module.decoder.layers.0.mlp.experts.linear_fc1.weight12
1332
  module.decoder.layers.0.mlp.experts.linear_fc1.weight10
1333
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight15
1334
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight13
1335
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight11
1336
  Params for bucket 66 (44040192 elements, 44040192 padded size):
1337
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight6
1338
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight8
1339
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight4
1340
  module.decoder.layers.0.mlp.experts.linear_fc1.weight9
1341
  module.decoder.layers.0.mlp.experts.linear_fc1.weight7
1342
  module.decoder.layers.0.mlp.experts.linear_fc1.weight5
 
 
 
1343
  Params for bucket 67 (29360128 elements, 29360128 padded size):
1344
  module.decoder.layers.0.mlp.experts.linear_fc1.weight3
1345
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight0
1346
  module.decoder.layers.0.mlp.experts.linear_fc1.weight1
1347
  module.decoder.layers.0.mlp.experts.linear_fc1.weight2
1348
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x7fa1fee3e780>, config_logger_dir='')
 
1349
  INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1350
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 05:35:27
1351
  > building train, validation, and test datasets ...
1352
  > datasets target sizes (minimum size):
1353
  train: 160
 
1355
  test: 0
1356
  INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
1357
  > building train, validation, and test datasets for GPT ...
1358
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x7fa20cc2a0c0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
1359
  INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
1360
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
1361
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
 
1368
  INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy
1369
  INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
1370
  > finished creating GPT datasets ...
1371
+ [after dataloaders are built] datetime: 2026-09-21 05:35:28
1372
  done with setup ...
 
 
 
1373
  training ...
1374
+ (min, max) time across ranks (ms):
1375
+ model-and-optimizer-setup ......................: (3615.62, 3625.22)
1376
+ train/valid/test-data-iterators-setup ..........: (102.03, 132.63)
1377
  Overwriting rerun_state_machine.current_iteration from -1 to 0...
1378
+ [before the start of training step] datetime: 2026-09-21 05:35:28
1379
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1380
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1381
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1382
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1383
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1384
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
 
 
1385
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1386
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1387
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1388
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
 
 
1389
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1390
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1391
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
 
1394
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1395
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1396
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1397
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1398
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1399
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1400
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1401
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1402
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1403
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
 
1428
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1429
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1430
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1431
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1432
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1433
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1434
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1435
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1436
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
 
 
1437
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1438
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1439
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
 
1504
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1505
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1506
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1507
+ [2026-09-21 05:36:00] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 31968.3 | throughput per GPU (TFLOP/s/GPU): 2.3 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102242E+01 | loss scale: 1.0 | grad norm: 49.126 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9600 | min safe capacity: 1.1719 |
1508
  Number of parameters in transformer block in billions: 23.22
1509
  Number of parameters in embedding layers in billions: 0.41
1510
  Total number of parameters in billions: 23.63
1511
  Number of parameters in most loaded shard in billions: 23.6320
1512
  Theoretical memory footprints: weight and optimizer=169029.43 MB
1513
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 58106.197265625 | max allocated: 64089.494140625 | reserved: 73358.0 | max reserved: 73358.0
1514
+ [2026-09-21 05:36:00] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 811.0 | throughput per GPU (TFLOP/s/GPU): 90.8 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101875E+01 | loss scale: 1.0 | grad norm: 51.017 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1515
+ [2026-09-21 05:36:01] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 463.0 | throughput per GPU (TFLOP/s/GPU): 159.0 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.011688E+01 | loss scale: 1.0 | grad norm: 160.654 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1516
  Capture CUDA graph for training!!!
1517
  CUDA graph capture done!!!
1518
+ [2026-09-21 05:36:04] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 3370.4 | throughput per GPU (TFLOP/s/GPU): 21.8 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.298903E+01 | loss scale: 1.0 | grad norm: 50.885 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1519
+ [2026-09-21 05:36:04] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 219.9 | throughput per GPU (TFLOP/s/GPU): 334.8 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.041365E+01 | loss scale: 1.0 | grad norm: 24.553 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1520
+ [2026-09-21 05:36:05] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 218.5 | throughput per GPU (TFLOP/s/GPU): 337.0 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.003537E+01 | loss scale: 1.0 | grad norm: 14.400 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1521
+ [2026-09-21 05:36:08] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3081.7 | throughput per GPU (TFLOP/s/GPU): 23.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.518715E+00 | loss scale: 1.0 | grad norm: 10.106 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1522
+ [2026-09-21 05:36:08] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 230.8 | throughput per GPU (TFLOP/s/GPU): 319.0 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.968018E+00 | loss scale: 1.0 | grad norm: 8.855 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1523
  Capture range started in the application.
1524
  Capture range ended in the application.
1525
+ Generating '/tmp/nsys-report-4e4d.qdstrm'
1526
 
1527
+
1528
+ [2026-09-21 05:36:51] iteration 9/ 10 | consumed samples: 144 | elapsed time per iteration (ms): 43450.9 | throughput per GPU (TFLOP/s/GPU): 1.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.729088E+00 | loss scale: 1.0 | grad norm: 7.909 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1529
+ [2026-09-21 05:36:52] iteration 10/ 10 | consumed samples: 160 | elapsed time per iteration (ms): 252.4 | throughput per GPU (TFLOP/s/GPU): 291.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.601188E+00 | loss scale: 1.0 | grad norm: 7.520 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 9728 | min safe capacity: 1.1875 |
1530
+ [after training is done] datetime: 2026-09-21 05:36:52
1531
+ [rank0]:[W921 05:36:53.476462482 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1532
+ [rank1]:[W921 05:36:54.800249819 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1533
+ [rank4]:[W921 05:36:54.800539743 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1534
+ [rank2]:[W921 05:36:54.801410824 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1535
+ [rank3]:[W921 05:36:54.802887482 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1536
+ [rank5]:[W921 05:36:54.807266313 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1537
+ [rank6]:[W921 05:36:54.808555910 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1538
+ [rank7]:[W921 05:36:55.183361105 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1539
  Collection stop is not allowed in this state.
1540
+ [rank3]:[W921 05:36:57.905144619 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
1541
+ pure virtual method called
1542
+ terminate called without an active exception
1543
+ [rank0]:[W921 05:36:58.989119651 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
1544
+ [rank4]:[W921 05:36:58.203681109 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
1545
+ pure virtual method called
1546
+ terminate called without an active exception
1547
+ [rank2]:[W921 05:36:58.384630062 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
1548
+ pure virtual method called
1549
+ terminate called without an active exception
1550
+ W0921 05:36:58.818000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64479 closing signal SIGTERM
1551
+ W0921 05:36:58.819000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64480 closing signal SIGTERM
1552
+ W0921 05:36:58.819000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64481 closing signal SIGTERM
1553
+ W0921 05:36:58.819000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64483 closing signal SIGTERM
1554
+ W0921 05:36:58.820000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64484 closing signal SIGTERM
1555
+ W0921 05:36:58.820000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64485 closing signal SIGTERM
1556
+ W0921 05:36:58.820000 64368 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 64486 closing signal SIGTERM
1557
+ E0921 05:37:01.353000 64368 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 3 (pid: 64482) of binary: /usr/bin/python
1558
  Traceback (most recent call last):
1559
  File "/usr/local/bin/torchrun", line 8, in <module>
1560
  sys.exit(main())
 
1569
  File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1570
  return launch_agent(self._config, self._entrypoint, list(args))
1571
  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1572
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
1573
+ raise ChildFailedError(
1574
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
1575
+ ============================================================
1576
+ pretrain_gpt.py FAILED
1577
+ ------------------------------------------------------------
1578
+ Failures:
1579
+ <NO_OTHER_FAILURES>
1580
+ ------------------------------------------------------------
1581
+ Root Cause (first observed failure):
1582
+ [0]:
1583
+ time : 2026-09-21_05:36:58
1584
+ host : sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
1585
+ rank : 3 (local_rank: 3)
1586
+ exitcode : -6 (pid: 64482)
1587
+ error_file: <N/A>
1588
+ traceback : Signal 6 (SIGABRT) received by PID 64482
1589
+ ============================================================
1590
+ Generated:
1591
+ /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cb0fab9880db3704485df86f5152aa7d22d61b36e8520c33c503bf109e2ec9af
3
+ size 14619213
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/train.log CHANGED
@@ -6,10 +6,10 @@
6
  memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/memory_snapshots/snapshot.pickle
7
  repo=/workspace/Megatron-UltraEP @ 03d57ef7
8
  ==============================================================
9
- W0917 03:39:00.688000 989911 torch/distributed/run.py:766]
10
- W0917 03:39:00.688000 989911 torch/distributed/run.py:766] *****************************************
11
- W0917 03:39:00.688000 989911 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
- W0917 03:39:00.688000 989911 torch/distributed/run.py:766] *****************************************
13
  using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
14
  WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
15
  Number of virtual stages per pipeline stage: None
@@ -574,28 +574,27 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode
574
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
575
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
576
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
 
577
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
578
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
579
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
580
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
581
- WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
582
- [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
583
- [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
  [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
- [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
- [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
- [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
  [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
  [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
- [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
593
  [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
594
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
- [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
597
  [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
- [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
599
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
601
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
@@ -606,24 +605,25 @@ WARNING: one_logger package is required to enable e2e metrics tracking. please g
606
  > initialized tensor model parallel with size 1
607
  > initialized pipeline model parallel with size 1
608
  > setting random seeds to 1234 ...
609
- [rank5]:[W917 03:39:13.781925838 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
- [rank7]:[W917 03:39:13.782013200 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
- [rank6]:[W917 03:39:13.782017160 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
- [rank1]:[W917 03:39:13.782380098 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
- [rank4]:[W917 03:39:13.782414970 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
- [rank2]:[W917 03:39:13.782547558 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
- [rank3]:[W917 03:39:13.785685160 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
 
616
  > compiling dataset index builder ...
617
  make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
618
  make: Nothing to be done for 'default'.
619
  make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
620
- >>> done with dataset index builder. Compilation time: 0.431 seconds
621
  WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
622
  > compiling and loading fused kernels ...
623
- [rank0]:[W917 03:39:14.411528119 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
624
- >>> done with compiling and loading fused kernels. Compilation time: 3.909 seconds
625
- time to initialize megatron (seconds): 7.647
626
- [after megatron is initialized] datetime: 2026-09-17 03:39:20
627
  building GPT model ...
628
  /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
629
  warnings.warn(
@@ -674,77 +674,77 @@ Params for bucket 2 (42479616 elements, 42479616 padded size):
674
  module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
675
  module.decoder.layers.15.self_attention.linear_proj.weight
676
  Params for bucket 3 (42475520 elements, 42475520 padded size):
677
- module.decoder.layers.14.self_attention.linear_qkv.weight
678
- module.decoder.layers.14.pre_mlp_layernorm.weight
679
  module.decoder.layers.14.mlp.router.weight
680
- module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
681
  module.decoder.layers.14.self_attention.linear_proj.weight
 
 
 
682
  Params for bucket 4 (42475520 elements, 42475520 padded size):
683
  module.decoder.layers.13.mlp.router.weight
684
- module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
685
  module.decoder.layers.13.self_attention.linear_proj.weight
 
686
  module.decoder.layers.13.self_attention.linear_qkv.weight
687
  module.decoder.layers.13.pre_mlp_layernorm.weight
688
  Params for bucket 5 (42475520 elements, 42475520 padded size):
689
  module.decoder.layers.12.self_attention.linear_qkv.weight
690
  module.decoder.layers.12.pre_mlp_layernorm.weight
691
  module.decoder.layers.12.mlp.router.weight
692
- module.decoder.layers.12.self_attention.linear_proj.weight
693
  module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
 
694
  Params for bucket 6 (42475520 elements, 42475520 padded size):
695
- module.decoder.layers.11.self_attention.linear_qkv.weight
696
  module.decoder.layers.11.pre_mlp_layernorm.weight
 
697
  module.decoder.layers.11.mlp.router.weight
698
  module.decoder.layers.11.self_attention.linear_proj.weight
699
  module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
700
  Params for bucket 7 (42475520 elements, 42475520 padded size):
 
 
701
  module.decoder.layers.10.mlp.router.weight
702
  module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
703
  module.decoder.layers.10.self_attention.linear_proj.weight
704
- module.decoder.layers.10.self_attention.linear_qkv.weight
705
- module.decoder.layers.10.pre_mlp_layernorm.weight
706
  Params for bucket 8 (42475520 elements, 42475520 padded size):
 
 
707
  module.decoder.layers.9.mlp.router.weight
708
  module.decoder.layers.9.self_attention.linear_proj.weight
709
  module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
710
- module.decoder.layers.9.pre_mlp_layernorm.weight
711
- module.decoder.layers.9.self_attention.linear_qkv.weight
712
  Params for bucket 9 (42475520 elements, 42475520 padded size):
 
 
713
  module.decoder.layers.8.mlp.router.weight
714
  module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
715
  module.decoder.layers.8.self_attention.linear_proj.weight
716
- module.decoder.layers.8.self_attention.linear_qkv.weight
717
- module.decoder.layers.8.pre_mlp_layernorm.weight
718
  Params for bucket 10 (42475520 elements, 42475520 padded size):
 
 
719
  module.decoder.layers.7.mlp.router.weight
720
  module.decoder.layers.7.self_attention.linear_proj.weight
721
  module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
722
- module.decoder.layers.7.pre_mlp_layernorm.weight
723
- module.decoder.layers.7.self_attention.linear_qkv.weight
724
  Params for bucket 11 (42475520 elements, 42475520 padded size):
725
- module.decoder.layers.6.pre_mlp_layernorm.weight
726
- module.decoder.layers.6.self_attention.linear_qkv.weight
727
  module.decoder.layers.6.mlp.router.weight
728
  module.decoder.layers.6.self_attention.linear_proj.weight
729
  module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
 
 
730
  Params for bucket 12 (42475520 elements, 42475520 padded size):
 
 
731
  module.decoder.layers.5.self_attention.linear_qkv.weight
732
  module.decoder.layers.5.self_attention.linear_proj.weight
733
  module.decoder.layers.5.pre_mlp_layernorm.weight
734
- module.decoder.layers.5.mlp.router.weight
735
- module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
736
  Params for bucket 13 (42475520 elements, 42475520 padded size):
737
- module.decoder.layers.4.mlp.router.weight
738
- module.decoder.layers.4.self_attention.linear_proj.weight
739
- module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
740
  module.decoder.layers.4.self_attention.linear_qkv.weight
741
  module.decoder.layers.4.pre_mlp_layernorm.weight
 
 
 
742
  Params for bucket 14 (42475520 elements, 42475520 padded size):
 
 
743
  module.decoder.layers.3.mlp.router.weight
744
  module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
745
  module.decoder.layers.3.self_attention.linear_proj.weight
746
- module.decoder.layers.3.self_attention.linear_qkv.weight
747
- module.decoder.layers.3.pre_mlp_layernorm.weight
748
  Params for bucket 15 (42475520 elements, 42475520 padded size):
749
  module.decoder.layers.2.mlp.router.weight
750
  module.decoder.layers.2.self_attention.linear_proj.weight
@@ -752,40 +752,40 @@ Params for bucket 15 (42475520 elements, 42475520 padded size):
752
  module.decoder.layers.2.pre_mlp_layernorm.weight
753
  module.decoder.layers.2.self_attention.linear_qkv.weight
754
  Params for bucket 16 (42475520 elements, 42475520 padded size):
755
- module.decoder.layers.1.self_attention.linear_qkv.weight
756
- module.decoder.layers.1.pre_mlp_layernorm.weight
757
  module.decoder.layers.1.mlp.router.weight
758
- module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
759
  module.decoder.layers.1.self_attention.linear_proj.weight
 
 
 
760
  Params for bucket 17 (42475520 elements, 42475520 padded size):
761
  module.decoder.layers.0.mlp.router.weight
762
  module.decoder.layers.0.pre_mlp_layernorm.weight
763
- module.decoder.layers.0.self_attention.linear_proj.weight
764
  module.decoder.layers.0.self_attention.linear_qkv.weight
 
765
  module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
766
  Params for bucket 18 (206045184 elements, 206045184 padded size):
767
  module.embedding.word_embeddings.weight
768
  INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67
769
  Params for bucket 1 (40370176 elements, 40370176 padded size):
770
- module.decoder.layers.15.mlp.experts.linear_fc2.weight12
771
- module.decoder.layers.15.mlp.experts.linear_fc2.weight14
772
- module.decoder.layers.15.mlp.experts.linear_fc2.weight10
773
- module.decoder.layers.15.mlp.experts.linear_fc2.weight8
774
- module.decoder.layers.15.mlp.experts.linear_fc2.weight6
775
  module.decoder.layers.15.mlp.experts.linear_fc2.weight15
776
  module.decoder.layers.15.mlp.experts.linear_fc2.weight13
777
  module.decoder.layers.15.mlp.experts.linear_fc2.weight11
778
  module.decoder.layers.15.mlp.experts.linear_fc2.weight9
779
  module.decoder.layers.15.mlp.experts.linear_fc2.weight7
780
  module.decoder.layers.15.mlp.experts.linear_fc2.weight5
 
 
 
 
 
781
  Params for bucket 2 (40370176 elements, 40370176 padded size):
782
- module.decoder.layers.15.mlp.experts.linear_fc2.weight4
783
- module.decoder.layers.15.mlp.experts.linear_fc2.weight2
784
- module.decoder.layers.15.mlp.experts.linear_fc2.weight0
785
- module.decoder.layers.15.mlp.experts.linear_fc1.weight15
786
- module.decoder.layers.15.mlp.experts.linear_fc1.weight13
787
  module.decoder.layers.15.mlp.experts.linear_fc2.weight3
788
  module.decoder.layers.15.mlp.experts.linear_fc2.weight1
 
 
 
 
 
789
  module.decoder.layers.15.mlp.experts.linear_fc1.weight14
790
  Params for bucket 3 (44040192 elements, 44040192 padded size):
791
  module.decoder.layers.15.mlp.experts.linear_fc1.weight11
@@ -799,16 +799,16 @@ Params for bucket 4 (44040192 elements, 44040192 padded size):
799
  module.decoder.layers.15.mlp.experts.linear_fc1.weight3
800
  module.decoder.layers.15.mlp.experts.linear_fc1.weight1
801
  module.decoder.layers.15.mlp.experts.linear_fc1.weight6
802
- module.decoder.layers.15.mlp.experts.linear_fc1.weight2
803
  module.decoder.layers.15.mlp.experts.linear_fc1.weight4
 
804
  Params for bucket 5 (40370176 elements, 40370176 padded size):
805
- module.decoder.layers.14.mlp.experts.linear_fc2.weight14
806
  module.decoder.layers.14.mlp.experts.linear_fc2.weight12
 
807
  module.decoder.layers.14.mlp.experts.linear_fc2.weight10
808
  module.decoder.layers.14.mlp.experts.linear_fc2.weight8
809
  module.decoder.layers.15.mlp.experts.linear_fc1.weight0
810
- module.decoder.layers.14.mlp.experts.linear_fc2.weight15
811
  module.decoder.layers.14.mlp.experts.linear_fc2.weight13
 
812
  module.decoder.layers.14.mlp.experts.linear_fc2.weight11
813
  module.decoder.layers.14.mlp.experts.linear_fc2.weight9
814
  module.decoder.layers.14.mlp.experts.linear_fc2.weight7
@@ -818,71 +818,71 @@ Params for bucket 6 (40370176 elements, 40370176 padded size):
818
  module.decoder.layers.14.mlp.experts.linear_fc2.weight2
819
  module.decoder.layers.14.mlp.experts.linear_fc2.weight0
820
  module.decoder.layers.14.mlp.experts.linear_fc1.weight15
821
- module.decoder.layers.14.mlp.experts.linear_fc2.weight5
822
  module.decoder.layers.14.mlp.experts.linear_fc2.weight3
 
823
  module.decoder.layers.14.mlp.experts.linear_fc2.weight1
824
  module.decoder.layers.14.mlp.experts.linear_fc1.weight14
825
  Params for bucket 7 (44040192 elements, 44040192 padded size):
826
  module.decoder.layers.14.mlp.experts.linear_fc1.weight13
827
- module.decoder.layers.14.mlp.experts.linear_fc1.weight11
828
  module.decoder.layers.14.mlp.experts.linear_fc1.weight9
 
829
  module.decoder.layers.14.mlp.experts.linear_fc1.weight10
830
  module.decoder.layers.14.mlp.experts.linear_fc1.weight12
831
  module.decoder.layers.14.mlp.experts.linear_fc1.weight8
832
  Params for bucket 8 (44040192 elements, 44040192 padded size):
833
  module.decoder.layers.14.mlp.experts.linear_fc1.weight7
834
- module.decoder.layers.14.mlp.experts.linear_fc1.weight5
835
  module.decoder.layers.14.mlp.experts.linear_fc1.weight3
836
- module.decoder.layers.14.mlp.experts.linear_fc1.weight4
837
  module.decoder.layers.14.mlp.experts.linear_fc1.weight6
 
838
  module.decoder.layers.14.mlp.experts.linear_fc1.weight2
839
  Params for bucket 9 (40370176 elements, 40370176 padded size):
840
  module.decoder.layers.14.mlp.experts.linear_fc1.weight1
841
- module.decoder.layers.13.mlp.experts.linear_fc2.weight14
842
- module.decoder.layers.13.mlp.experts.linear_fc2.weight12
843
- module.decoder.layers.13.mlp.experts.linear_fc2.weight10
844
- module.decoder.layers.14.mlp.experts.linear_fc1.weight0
845
  module.decoder.layers.13.mlp.experts.linear_fc2.weight15
846
  module.decoder.layers.13.mlp.experts.linear_fc2.weight13
847
  module.decoder.layers.13.mlp.experts.linear_fc2.weight11
848
  module.decoder.layers.13.mlp.experts.linear_fc2.weight9
 
 
 
 
849
  Params for bucket 10 (40370176 elements, 40370176 padded size):
 
 
 
 
850
  module.decoder.layers.13.mlp.experts.linear_fc2.weight8
851
  module.decoder.layers.13.mlp.experts.linear_fc2.weight4
852
  module.decoder.layers.13.mlp.experts.linear_fc2.weight6
853
  module.decoder.layers.13.mlp.experts.linear_fc2.weight2
854
  module.decoder.layers.13.mlp.experts.linear_fc2.weight0
855
- module.decoder.layers.13.mlp.experts.linear_fc2.weight7
856
- module.decoder.layers.13.mlp.experts.linear_fc2.weight5
857
- module.decoder.layers.13.mlp.experts.linear_fc2.weight3
858
- module.decoder.layers.13.mlp.experts.linear_fc2.weight1
859
  module.decoder.layers.13.mlp.experts.linear_fc1.weight15
860
  Params for bucket 11 (44040192 elements, 44040192 padded size):
861
  module.decoder.layers.13.mlp.experts.linear_fc1.weight14
862
  module.decoder.layers.13.mlp.experts.linear_fc1.weight10
863
  module.decoder.layers.13.mlp.experts.linear_fc1.weight12
864
- module.decoder.layers.13.mlp.experts.linear_fc1.weight11
865
  module.decoder.layers.13.mlp.experts.linear_fc1.weight13
 
866
  module.decoder.layers.13.mlp.experts.linear_fc1.weight9
867
  Params for bucket 12 (44040192 elements, 44040192 padded size):
868
  module.decoder.layers.13.mlp.experts.linear_fc1.weight8
869
  module.decoder.layers.13.mlp.experts.linear_fc1.weight4
870
  module.decoder.layers.13.mlp.experts.linear_fc1.weight6
871
- module.decoder.layers.13.mlp.experts.linear_fc1.weight5
872
  module.decoder.layers.13.mlp.experts.linear_fc1.weight7
 
873
  module.decoder.layers.13.mlp.experts.linear_fc1.weight3
874
  Params for bucket 13 (40370176 elements, 40370176 padded size):
875
  module.decoder.layers.13.mlp.experts.linear_fc1.weight2
876
- module.decoder.layers.12.mlp.experts.linear_fc2.weight15
877
  module.decoder.layers.13.mlp.experts.linear_fc1.weight0
 
878
  module.decoder.layers.12.mlp.experts.linear_fc2.weight13
879
  module.decoder.layers.12.mlp.experts.linear_fc2.weight11
880
  module.decoder.layers.13.mlp.experts.linear_fc1.weight1
881
  module.decoder.layers.12.mlp.experts.linear_fc2.weight14
882
  module.decoder.layers.12.mlp.experts.linear_fc2.weight12
883
  Params for bucket 14 (40370176 elements, 40370176 padded size):
884
- module.decoder.layers.12.mlp.experts.linear_fc2.weight9
885
  module.decoder.layers.12.mlp.experts.linear_fc2.weight7
 
886
  module.decoder.layers.12.mlp.experts.linear_fc2.weight5
887
  module.decoder.layers.12.mlp.experts.linear_fc2.weight3
888
  module.decoder.layers.12.mlp.experts.linear_fc2.weight1
@@ -893,282 +893,282 @@ Params for bucket 14 (40370176 elements, 40370176 padded size):
893
  module.decoder.layers.12.mlp.experts.linear_fc2.weight2
894
  module.decoder.layers.12.mlp.experts.linear_fc2.weight0
895
  Params for bucket 15 (44040192 elements, 44040192 padded size):
896
- module.decoder.layers.12.mlp.experts.linear_fc1.weight12
897
- module.decoder.layers.12.mlp.experts.linear_fc1.weight14
898
- module.decoder.layers.12.mlp.experts.linear_fc1.weight10
899
  module.decoder.layers.12.mlp.experts.linear_fc1.weight15
900
  module.decoder.layers.12.mlp.experts.linear_fc1.weight11
901
  module.decoder.layers.12.mlp.experts.linear_fc1.weight13
 
 
 
902
  Params for bucket 16 (44040192 elements, 44040192 padded size):
903
- module.decoder.layers.12.mlp.experts.linear_fc1.weight6
904
- module.decoder.layers.12.mlp.experts.linear_fc1.weight8
905
- module.decoder.layers.12.mlp.experts.linear_fc1.weight4
906
  module.decoder.layers.12.mlp.experts.linear_fc1.weight9
907
  module.decoder.layers.12.mlp.experts.linear_fc1.weight7
908
  module.decoder.layers.12.mlp.experts.linear_fc1.weight5
 
 
 
909
  Params for bucket 17 (40370176 elements, 40370176 padded size):
910
- module.decoder.layers.12.mlp.experts.linear_fc1.weight0
911
- module.decoder.layers.12.mlp.experts.linear_fc1.weight2
912
- module.decoder.layers.11.mlp.experts.linear_fc2.weight14
913
  module.decoder.layers.12.mlp.experts.linear_fc1.weight3
914
- module.decoder.layers.11.mlp.experts.linear_fc2.weight15
915
  module.decoder.layers.12.mlp.experts.linear_fc1.weight1
 
 
 
 
916
  module.decoder.layers.11.mlp.experts.linear_fc2.weight13
917
  Params for bucket 18 (40370176 elements, 40370176 padded size):
918
  module.decoder.layers.11.mlp.experts.linear_fc2.weight12
919
- module.decoder.layers.11.mlp.experts.linear_fc2.weight10
920
  module.decoder.layers.11.mlp.experts.linear_fc2.weight8
 
921
  module.decoder.layers.11.mlp.experts.linear_fc2.weight6
922
  module.decoder.layers.11.mlp.experts.linear_fc2.weight4
923
  module.decoder.layers.11.mlp.experts.linear_fc2.weight2
924
- module.decoder.layers.11.mlp.experts.linear_fc2.weight9
925
  module.decoder.layers.11.mlp.experts.linear_fc2.weight11
 
926
  module.decoder.layers.11.mlp.experts.linear_fc2.weight7
927
  module.decoder.layers.11.mlp.experts.linear_fc2.weight5
928
  module.decoder.layers.11.mlp.experts.linear_fc2.weight3
929
  Params for bucket 19 (44040192 elements, 44040192 padded size):
 
930
  module.decoder.layers.11.mlp.experts.linear_fc1.weight13
931
  module.decoder.layers.11.mlp.experts.linear_fc2.weight0
932
- module.decoder.layers.11.mlp.experts.linear_fc1.weight15
933
  module.decoder.layers.11.mlp.experts.linear_fc1.weight11
934
  module.decoder.layers.11.mlp.experts.linear_fc2.weight1
935
  module.decoder.layers.11.mlp.experts.linear_fc1.weight14
936
  module.decoder.layers.11.mlp.experts.linear_fc1.weight12
937
  Params for bucket 20 (44040192 elements, 44040192 padded size):
938
- module.decoder.layers.11.mlp.experts.linear_fc1.weight7
939
  module.decoder.layers.11.mlp.experts.linear_fc1.weight9
 
940
  module.decoder.layers.11.mlp.experts.linear_fc1.weight5
941
  module.decoder.layers.11.mlp.experts.linear_fc1.weight10
942
- module.decoder.layers.11.mlp.experts.linear_fc1.weight8
943
  module.decoder.layers.11.mlp.experts.linear_fc1.weight6
 
944
  Params for bucket 21 (40370176 elements, 40370176 padded size):
945
  module.decoder.layers.11.mlp.experts.linear_fc1.weight3
946
  module.decoder.layers.11.mlp.experts.linear_fc1.weight1
947
- module.decoder.layers.10.mlp.experts.linear_fc2.weight15
948
  module.decoder.layers.11.mlp.experts.linear_fc1.weight4
949
- module.decoder.layers.11.mlp.experts.linear_fc1.weight0
950
  module.decoder.layers.11.mlp.experts.linear_fc1.weight2
 
 
951
  Params for bucket 22 (40370176 elements, 40370176 padded size):
952
- module.decoder.layers.10.mlp.experts.linear_fc2.weight13
953
- module.decoder.layers.10.mlp.experts.linear_fc2.weight11
954
- module.decoder.layers.10.mlp.experts.linear_fc2.weight9
955
- module.decoder.layers.10.mlp.experts.linear_fc2.weight7
956
- module.decoder.layers.10.mlp.experts.linear_fc2.weight5
957
  module.decoder.layers.10.mlp.experts.linear_fc2.weight14
958
  module.decoder.layers.10.mlp.experts.linear_fc2.weight10
959
  module.decoder.layers.10.mlp.experts.linear_fc2.weight12
960
  module.decoder.layers.10.mlp.experts.linear_fc2.weight8
961
  module.decoder.layers.10.mlp.experts.linear_fc2.weight6
962
  module.decoder.layers.10.mlp.experts.linear_fc2.weight4
 
 
 
 
 
963
  Params for bucket 23 (44040192 elements, 44040192 padded size):
964
- module.decoder.layers.10.mlp.experts.linear_fc2.weight3
965
- module.decoder.layers.10.mlp.experts.linear_fc2.weight1
966
- module.decoder.layers.10.mlp.experts.linear_fc1.weight14
967
- module.decoder.layers.10.mlp.experts.linear_fc1.weight12
968
  module.decoder.layers.10.mlp.experts.linear_fc2.weight0
969
  module.decoder.layers.10.mlp.experts.linear_fc1.weight15
970
  module.decoder.layers.10.mlp.experts.linear_fc2.weight2
971
  module.decoder.layers.10.mlp.experts.linear_fc1.weight13
 
 
 
 
972
  Params for bucket 24 (44040192 elements, 44040192 padded size):
973
- module.decoder.layers.10.mlp.experts.linear_fc1.weight10
974
- module.decoder.layers.10.mlp.experts.linear_fc1.weight8
975
- module.decoder.layers.10.mlp.experts.linear_fc1.weight6
976
  module.decoder.layers.10.mlp.experts.linear_fc1.weight11
977
  module.decoder.layers.10.mlp.experts.linear_fc1.weight7
978
  module.decoder.layers.10.mlp.experts.linear_fc1.weight9
 
 
 
979
  Params for bucket 25 (44040192 elements, 44040192 padded size):
980
- module.decoder.layers.10.mlp.experts.linear_fc1.weight4
981
- module.decoder.layers.10.mlp.experts.linear_fc1.weight2
982
- module.decoder.layers.10.mlp.experts.linear_fc1.weight0
983
  module.decoder.layers.10.mlp.experts.linear_fc1.weight5
984
  module.decoder.layers.10.mlp.experts.linear_fc1.weight1
985
  module.decoder.layers.10.mlp.experts.linear_fc1.weight3
 
 
 
986
  Params for bucket 26 (40370176 elements, 40370176 padded size):
 
 
 
 
 
987
  module.decoder.layers.9.mlp.experts.linear_fc2.weight15
988
  module.decoder.layers.9.mlp.experts.linear_fc2.weight11
989
  module.decoder.layers.9.mlp.experts.linear_fc2.weight13
990
  module.decoder.layers.9.mlp.experts.linear_fc2.weight9
991
  module.decoder.layers.9.mlp.experts.linear_fc2.weight7
992
  module.decoder.layers.9.mlp.experts.linear_fc2.weight5
993
- module.decoder.layers.9.mlp.experts.linear_fc2.weight12
994
- module.decoder.layers.9.mlp.experts.linear_fc2.weight14
995
- module.decoder.layers.9.mlp.experts.linear_fc2.weight10
996
- module.decoder.layers.9.mlp.experts.linear_fc2.weight8
997
- module.decoder.layers.9.mlp.experts.linear_fc2.weight6
998
  Params for bucket 27 (40370176 elements, 40370176 padded size):
999
- module.decoder.layers.9.mlp.experts.linear_fc2.weight1
1000
- module.decoder.layers.9.mlp.experts.linear_fc2.weight3
1001
- module.decoder.layers.9.mlp.experts.linear_fc1.weight14
1002
  module.decoder.layers.9.mlp.experts.linear_fc2.weight4
1003
  module.decoder.layers.9.mlp.experts.linear_fc2.weight2
1004
  module.decoder.layers.9.mlp.experts.linear_fc2.weight0
1005
  module.decoder.layers.9.mlp.experts.linear_fc1.weight15
1006
  module.decoder.layers.9.mlp.experts.linear_fc1.weight13
 
 
 
1007
  Params for bucket 28 (44040192 elements, 44040192 padded size):
1008
- module.decoder.layers.9.mlp.experts.linear_fc1.weight12
1009
- module.decoder.layers.9.mlp.experts.linear_fc1.weight8
1010
- module.decoder.layers.9.mlp.experts.linear_fc1.weight10
1011
  module.decoder.layers.9.mlp.experts.linear_fc1.weight9
1012
  module.decoder.layers.9.mlp.experts.linear_fc1.weight11
1013
  module.decoder.layers.9.mlp.experts.linear_fc1.weight7
 
 
 
1014
  Params for bucket 29 (44040192 elements, 44040192 padded size):
1015
- module.decoder.layers.9.mlp.experts.linear_fc1.weight6
1016
- module.decoder.layers.9.mlp.experts.linear_fc1.weight2
1017
- module.decoder.layers.9.mlp.experts.linear_fc1.weight4
1018
  module.decoder.layers.9.mlp.experts.linear_fc1.weight3
1019
  module.decoder.layers.9.mlp.experts.linear_fc1.weight5
1020
  module.decoder.layers.9.mlp.experts.linear_fc1.weight1
 
 
 
1021
  Params for bucket 30 (40370176 elements, 40370176 padded size):
 
 
 
 
1022
  module.decoder.layers.9.mlp.experts.linear_fc1.weight0
1023
  module.decoder.layers.8.mlp.experts.linear_fc2.weight13
1024
  module.decoder.layers.8.mlp.experts.linear_fc2.weight15
1025
  module.decoder.layers.8.mlp.experts.linear_fc2.weight11
1026
  module.decoder.layers.8.mlp.experts.linear_fc2.weight9
1027
  module.decoder.layers.8.mlp.experts.linear_fc2.weight7
1028
- module.decoder.layers.8.mlp.experts.linear_fc2.weight14
1029
- module.decoder.layers.8.mlp.experts.linear_fc2.weight12
1030
- module.decoder.layers.8.mlp.experts.linear_fc2.weight10
1031
- module.decoder.layers.8.mlp.experts.linear_fc2.weight8
1032
  Params for bucket 31 (40370176 elements, 40370176 padded size):
1033
- module.decoder.layers.8.mlp.experts.linear_fc2.weight3
1034
- module.decoder.layers.8.mlp.experts.linear_fc2.weight5
1035
- module.decoder.layers.8.mlp.experts.linear_fc2.weight1
1036
- module.decoder.layers.8.mlp.experts.linear_fc1.weight14
1037
  module.decoder.layers.8.mlp.experts.linear_fc2.weight6
1038
  module.decoder.layers.8.mlp.experts.linear_fc2.weight4
1039
  module.decoder.layers.8.mlp.experts.linear_fc2.weight2
1040
  module.decoder.layers.8.mlp.experts.linear_fc2.weight0
1041
  module.decoder.layers.8.mlp.experts.linear_fc1.weight15
 
 
 
 
1042
  Params for bucket 32 (44040192 elements, 44040192 padded size):
1043
- module.decoder.layers.8.mlp.experts.linear_fc1.weight10
1044
- module.decoder.layers.8.mlp.experts.linear_fc1.weight12
1045
- module.decoder.layers.8.mlp.experts.linear_fc1.weight8
1046
  module.decoder.layers.8.mlp.experts.linear_fc1.weight13
1047
  module.decoder.layers.8.mlp.experts.linear_fc1.weight11
1048
  module.decoder.layers.8.mlp.experts.linear_fc1.weight9
 
 
 
1049
  Params for bucket 33 (44040192 elements, 44040192 padded size):
 
 
 
1050
  module.decoder.layers.8.mlp.experts.linear_fc1.weight4
1051
  module.decoder.layers.8.mlp.experts.linear_fc1.weight6
1052
  module.decoder.layers.8.mlp.experts.linear_fc1.weight2
1053
- module.decoder.layers.8.mlp.experts.linear_fc1.weight7
1054
- module.decoder.layers.8.mlp.experts.linear_fc1.weight3
1055
- module.decoder.layers.8.mlp.experts.linear_fc1.weight5
1056
  Params for bucket 34 (40370176 elements, 40370176 padded size):
1057
- module.decoder.layers.7.mlp.experts.linear_fc2.weight15
1058
- module.decoder.layers.8.mlp.experts.linear_fc1.weight0
1059
- module.decoder.layers.7.mlp.experts.linear_fc2.weight13
1060
- module.decoder.layers.7.mlp.experts.linear_fc2.weight11
1061
- module.decoder.layers.7.mlp.experts.linear_fc2.weight9
1062
  module.decoder.layers.8.mlp.experts.linear_fc1.weight1
1063
  module.decoder.layers.7.mlp.experts.linear_fc2.weight14
1064
  module.decoder.layers.7.mlp.experts.linear_fc2.weight12
1065
  module.decoder.layers.7.mlp.experts.linear_fc2.weight10
1066
- Params for bucket 35 (40370176 elements, 40370176 padded size):
1067
- module.decoder.layers.7.mlp.experts.linear_fc2.weight5
1068
- module.decoder.layers.7.mlp.experts.linear_fc2.weight7
1069
- module.decoder.layers.7.mlp.experts.linear_fc2.weight3
1070
- module.decoder.layers.7.mlp.experts.linear_fc2.weight1
 
1071
  module.decoder.layers.7.mlp.experts.linear_fc2.weight8
1072
- module.decoder.layers.7.mlp.experts.linear_fc2.weight4
1073
  module.decoder.layers.7.mlp.experts.linear_fc2.weight6
 
1074
  module.decoder.layers.7.mlp.experts.linear_fc2.weight2
1075
  module.decoder.layers.7.mlp.experts.linear_fc2.weight0
1076
  module.decoder.layers.7.mlp.experts.linear_fc1.weight15
 
 
 
 
1077
  Params for bucket 36 (44040192 elements, 44040192 padded size):
1078
- module.decoder.layers.7.mlp.experts.linear_fc1.weight14
1079
- module.decoder.layers.7.mlp.experts.linear_fc1.weight12
1080
- module.decoder.layers.7.mlp.experts.linear_fc1.weight10
1081
  module.decoder.layers.7.mlp.experts.linear_fc1.weight13
1082
  module.decoder.layers.7.mlp.experts.linear_fc1.weight11
1083
  module.decoder.layers.7.mlp.experts.linear_fc1.weight9
 
 
 
1084
  Params for bucket 37 (44040192 elements, 44040192 padded size):
1085
- module.decoder.layers.7.mlp.experts.linear_fc1.weight8
1086
- module.decoder.layers.7.mlp.experts.linear_fc1.weight6
1087
- module.decoder.layers.7.mlp.experts.linear_fc1.weight4
1088
  module.decoder.layers.7.mlp.experts.linear_fc1.weight7
1089
  module.decoder.layers.7.mlp.experts.linear_fc1.weight5
1090
  module.decoder.layers.7.mlp.experts.linear_fc1.weight3
 
 
 
1091
  Params for bucket 38 (40370176 elements, 40370176 padded size):
 
 
 
1092
  module.decoder.layers.7.mlp.experts.linear_fc1.weight2
1093
- module.decoder.layers.7.mlp.experts.linear_fc1.weight0
1094
  module.decoder.layers.6.mlp.experts.linear_fc2.weight15
 
1095
  module.decoder.layers.6.mlp.experts.linear_fc2.weight13
1096
  module.decoder.layers.6.mlp.experts.linear_fc2.weight11
1097
- module.decoder.layers.6.mlp.experts.linear_fc2.weight14
1098
- module.decoder.layers.7.mlp.experts.linear_fc1.weight1
1099
- module.decoder.layers.6.mlp.experts.linear_fc2.weight12
1100
  Params for bucket 39 (40370176 elements, 40370176 padded size):
1101
- module.decoder.layers.6.mlp.experts.linear_fc2.weight7
1102
- module.decoder.layers.6.mlp.experts.linear_fc2.weight9
1103
- module.decoder.layers.6.mlp.experts.linear_fc2.weight5
1104
- module.decoder.layers.6.mlp.experts.linear_fc2.weight3
1105
- module.decoder.layers.6.mlp.experts.linear_fc2.weight1
1106
  module.decoder.layers.6.mlp.experts.linear_fc2.weight10
1107
  module.decoder.layers.6.mlp.experts.linear_fc2.weight6
1108
  module.decoder.layers.6.mlp.experts.linear_fc2.weight8
1109
  module.decoder.layers.6.mlp.experts.linear_fc2.weight4
1110
  module.decoder.layers.6.mlp.experts.linear_fc2.weight2
1111
  module.decoder.layers.6.mlp.experts.linear_fc2.weight0
 
 
 
 
 
1112
  Params for bucket 40 (44040192 elements, 44040192 padded size):
1113
- module.decoder.layers.6.mlp.experts.linear_fc1.weight15
1114
- module.decoder.layers.6.mlp.experts.linear_fc1.weight13
1115
- module.decoder.layers.6.mlp.experts.linear_fc1.weight11
1116
- module.decoder.layers.6.mlp.experts.linear_fc1.weight12
1117
  module.decoder.layers.6.mlp.experts.linear_fc1.weight14
 
1118
  module.decoder.layers.6.mlp.experts.linear_fc1.weight10
 
 
 
1119
  Params for bucket 41 (44040192 elements, 44040192 padded size):
1120
- module.decoder.layers.6.mlp.experts.linear_fc1.weight9
1121
- module.decoder.layers.6.mlp.experts.linear_fc1.weight7
1122
- module.decoder.layers.6.mlp.experts.linear_fc1.weight5
1123
  module.decoder.layers.6.mlp.experts.linear_fc1.weight6
1124
  module.decoder.layers.6.mlp.experts.linear_fc1.weight8
1125
  module.decoder.layers.6.mlp.experts.linear_fc1.weight4
 
 
 
1126
  Params for bucket 42 (40370176 elements, 40370176 padded size):
1127
- module.decoder.layers.6.mlp.experts.linear_fc1.weight3
1128
- module.decoder.layers.6.mlp.experts.linear_fc1.weight1
1129
- module.decoder.layers.5.mlp.experts.linear_fc2.weight14
1130
  module.decoder.layers.6.mlp.experts.linear_fc1.weight0
1131
  module.decoder.layers.5.mlp.experts.linear_fc2.weight15
1132
  module.decoder.layers.6.mlp.experts.linear_fc1.weight2
1133
  module.decoder.layers.5.mlp.experts.linear_fc2.weight13
 
 
 
1134
  Params for bucket 43 (40370176 elements, 40370176 padded size):
1135
- module.decoder.layers.5.mlp.experts.linear_fc2.weight12
1136
- module.decoder.layers.5.mlp.experts.linear_fc2.weight10
1137
- module.decoder.layers.5.mlp.experts.linear_fc2.weight8
1138
- module.decoder.layers.5.mlp.experts.linear_fc2.weight6
1139
- module.decoder.layers.5.mlp.experts.linear_fc2.weight4
1140
- module.decoder.layers.5.mlp.experts.linear_fc2.weight2
1141
  module.decoder.layers.5.mlp.experts.linear_fc2.weight11
1142
  module.decoder.layers.5.mlp.experts.linear_fc2.weight9
1143
  module.decoder.layers.5.mlp.experts.linear_fc2.weight7
1144
  module.decoder.layers.5.mlp.experts.linear_fc2.weight5
1145
  module.decoder.layers.5.mlp.experts.linear_fc2.weight3
 
 
 
 
 
 
1146
  Params for bucket 44 (44040192 elements, 44040192 padded size):
1147
- module.decoder.layers.5.mlp.experts.linear_fc2.weight0
1148
- module.decoder.layers.5.mlp.experts.linear_fc1.weight15
1149
- module.decoder.layers.5.mlp.experts.linear_fc1.weight13
1150
- module.decoder.layers.5.mlp.experts.linear_fc1.weight11
1151
  module.decoder.layers.5.mlp.experts.linear_fc2.weight1
1152
  module.decoder.layers.5.mlp.experts.linear_fc1.weight14
1153
  module.decoder.layers.5.mlp.experts.linear_fc1.weight12
 
 
 
 
1154
  Params for bucket 45 (44040192 elements, 44040192 padded size):
 
 
 
1155
  module.decoder.layers.5.mlp.experts.linear_fc1.weight9
1156
  module.decoder.layers.5.mlp.experts.linear_fc1.weight7
1157
  module.decoder.layers.5.mlp.experts.linear_fc1.weight5
1158
- module.decoder.layers.5.mlp.experts.linear_fc1.weight10
1159
- module.decoder.layers.5.mlp.experts.linear_fc1.weight6
1160
- module.decoder.layers.5.mlp.experts.linear_fc1.weight8
1161
  Params for bucket 46 (40370176 elements, 40370176 padded size):
1162
- module.decoder.layers.5.mlp.experts.linear_fc1.weight1
1163
- module.decoder.layers.5.mlp.experts.linear_fc1.weight3
1164
  module.decoder.layers.5.mlp.experts.linear_fc1.weight4
1165
  module.decoder.layers.5.mlp.experts.linear_fc1.weight2
1166
  module.decoder.layers.5.mlp.experts.linear_fc1.weight0
 
 
1167
  module.decoder.layers.4.mlp.experts.linear_fc2.weight15
1168
  Params for bucket 47 (40370176 elements, 40370176 padded size):
1169
  module.decoder.layers.4.mlp.experts.linear_fc2.weight14
1170
- module.decoder.layers.4.mlp.experts.linear_fc2.weight12
1171
  module.decoder.layers.4.mlp.experts.linear_fc2.weight10
 
1172
  module.decoder.layers.4.mlp.experts.linear_fc2.weight8
1173
  module.decoder.layers.4.mlp.experts.linear_fc2.weight6
1174
  module.decoder.layers.4.mlp.experts.linear_fc2.weight4
@@ -1178,83 +1178,83 @@ Params for bucket 47 (40370176 elements, 40370176 padded size):
1178
  module.decoder.layers.4.mlp.experts.linear_fc2.weight7
1179
  module.decoder.layers.4.mlp.experts.linear_fc2.weight5
1180
  Params for bucket 48 (44040192 elements, 44040192 padded size):
1181
- module.decoder.layers.4.mlp.experts.linear_fc2.weight2
1182
  module.decoder.layers.4.mlp.experts.linear_fc2.weight0
1183
- module.decoder.layers.4.mlp.experts.linear_fc1.weight14
1184
- module.decoder.layers.4.mlp.experts.linear_fc1.weight12
1185
- module.decoder.layers.4.mlp.experts.linear_fc2.weight3
1186
  module.decoder.layers.4.mlp.experts.linear_fc1.weight15
1187
- module.decoder.layers.4.mlp.experts.linear_fc2.weight1
1188
  module.decoder.layers.4.mlp.experts.linear_fc1.weight13
 
 
 
 
1189
  Params for bucket 49 (44040192 elements, 44040192 padded size):
1190
- module.decoder.layers.4.mlp.experts.linear_fc1.weight10
1191
- module.decoder.layers.4.mlp.experts.linear_fc1.weight8
1192
- module.decoder.layers.4.mlp.experts.linear_fc1.weight6
1193
  module.decoder.layers.4.mlp.experts.linear_fc1.weight11
1194
  module.decoder.layers.4.mlp.experts.linear_fc1.weight9
1195
  module.decoder.layers.4.mlp.experts.linear_fc1.weight7
 
 
 
1196
  Params for bucket 50 (44040192 elements, 44040192 padded size):
1197
- module.decoder.layers.4.mlp.experts.linear_fc1.weight2
1198
- module.decoder.layers.4.mlp.experts.linear_fc1.weight4
1199
- module.decoder.layers.4.mlp.experts.linear_fc1.weight0
1200
  module.decoder.layers.4.mlp.experts.linear_fc1.weight5
1201
  module.decoder.layers.4.mlp.experts.linear_fc1.weight1
1202
  module.decoder.layers.4.mlp.experts.linear_fc1.weight3
 
 
 
1203
  Params for bucket 51 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1204
  module.decoder.layers.3.mlp.experts.linear_fc2.weight15
1205
  module.decoder.layers.3.mlp.experts.linear_fc2.weight11
1206
  module.decoder.layers.3.mlp.experts.linear_fc2.weight13
1207
  module.decoder.layers.3.mlp.experts.linear_fc2.weight9
1208
  module.decoder.layers.3.mlp.experts.linear_fc2.weight7
1209
  module.decoder.layers.3.mlp.experts.linear_fc2.weight5
1210
- module.decoder.layers.3.mlp.experts.linear_fc2.weight14
1211
- module.decoder.layers.3.mlp.experts.linear_fc2.weight12
1212
- module.decoder.layers.3.mlp.experts.linear_fc2.weight10
1213
- module.decoder.layers.3.mlp.experts.linear_fc2.weight8
1214
- module.decoder.layers.3.mlp.experts.linear_fc2.weight6
1215
  Params for bucket 52 (40370176 elements, 40370176 padded size):
1216
- module.decoder.layers.3.mlp.experts.linear_fc2.weight1
1217
- module.decoder.layers.3.mlp.experts.linear_fc2.weight3
1218
- module.decoder.layers.3.mlp.experts.linear_fc1.weight14
1219
  module.decoder.layers.3.mlp.experts.linear_fc2.weight4
1220
- module.decoder.layers.3.mlp.experts.linear_fc2.weight0
1221
  module.decoder.layers.3.mlp.experts.linear_fc2.weight2
 
1222
  module.decoder.layers.3.mlp.experts.linear_fc1.weight15
1223
  module.decoder.layers.3.mlp.experts.linear_fc1.weight13
 
 
 
1224
  Params for bucket 53 (44040192 elements, 44040192 padded size):
1225
- module.decoder.layers.3.mlp.experts.linear_fc1.weight12
1226
- module.decoder.layers.3.mlp.experts.linear_fc1.weight10
1227
- module.decoder.layers.3.mlp.experts.linear_fc1.weight8
1228
  module.decoder.layers.3.mlp.experts.linear_fc1.weight11
1229
  module.decoder.layers.3.mlp.experts.linear_fc1.weight9
1230
  module.decoder.layers.3.mlp.experts.linear_fc1.weight7
 
 
 
1231
  Params for bucket 54 (44040192 elements, 44040192 padded size):
1232
- module.decoder.layers.3.mlp.experts.linear_fc1.weight6
1233
- module.decoder.layers.3.mlp.experts.linear_fc1.weight4
1234
- module.decoder.layers.3.mlp.experts.linear_fc1.weight2
1235
  module.decoder.layers.3.mlp.experts.linear_fc1.weight5
1236
  module.decoder.layers.3.mlp.experts.linear_fc1.weight3
1237
  module.decoder.layers.3.mlp.experts.linear_fc1.weight1
 
 
 
1238
  Params for bucket 55 (40370176 elements, 40370176 padded size):
 
 
 
 
1239
  module.decoder.layers.3.mlp.experts.linear_fc1.weight0
1240
- module.decoder.layers.2.mlp.experts.linear_fc2.weight15
1241
  module.decoder.layers.2.mlp.experts.linear_fc2.weight13
 
1242
  module.decoder.layers.2.mlp.experts.linear_fc2.weight11
1243
  module.decoder.layers.2.mlp.experts.linear_fc2.weight9
1244
  module.decoder.layers.2.mlp.experts.linear_fc2.weight7
1245
- module.decoder.layers.2.mlp.experts.linear_fc2.weight12
1246
- module.decoder.layers.2.mlp.experts.linear_fc2.weight14
1247
- module.decoder.layers.2.mlp.experts.linear_fc2.weight10
1248
- module.decoder.layers.2.mlp.experts.linear_fc2.weight8
1249
  Params for bucket 56 (40370176 elements, 40370176 padded size):
1250
- module.decoder.layers.2.mlp.experts.linear_fc2.weight3
1251
- module.decoder.layers.2.mlp.experts.linear_fc2.weight5
1252
- module.decoder.layers.2.mlp.experts.linear_fc2.weight1
1253
- module.decoder.layers.2.mlp.experts.linear_fc1.weight14
1254
  module.decoder.layers.2.mlp.experts.linear_fc2.weight6
1255
  module.decoder.layers.2.mlp.experts.linear_fc2.weight4
1256
  module.decoder.layers.2.mlp.experts.linear_fc2.weight2
1257
  module.decoder.layers.2.mlp.experts.linear_fc2.weight0
 
 
 
 
1258
  module.decoder.layers.2.mlp.experts.linear_fc1.weight15
1259
  Params for bucket 57 (44040192 elements, 44040192 padded size):
1260
  module.decoder.layers.2.mlp.experts.linear_fc1.weight12
@@ -1296,8 +1296,8 @@ Params for bucket 61 (44040192 elements, 44040192 padded size):
1296
  module.decoder.layers.1.mlp.experts.linear_fc1.weight11
1297
  module.decoder.layers.1.mlp.experts.linear_fc1.weight9
1298
  module.decoder.layers.1.mlp.experts.linear_fc1.weight14
1299
- module.decoder.layers.1.mlp.experts.linear_fc1.weight10
1300
  module.decoder.layers.1.mlp.experts.linear_fc1.weight12
 
1301
  Params for bucket 62 (44040192 elements, 44040192 padded size):
1302
  module.decoder.layers.1.mlp.experts.linear_fc1.weight7
1303
  module.decoder.layers.1.mlp.experts.linear_fc1.weight5
@@ -1306,48 +1306,48 @@ Params for bucket 62 (44040192 elements, 44040192 padded size):
1306
  module.decoder.layers.1.mlp.experts.linear_fc1.weight6
1307
  module.decoder.layers.1.mlp.experts.linear_fc1.weight4
1308
  Params for bucket 63 (40370176 elements, 40370176 padded size):
1309
- module.decoder.layers.0.mlp.experts.linear_fc2.weight14
1310
  module.decoder.layers.1.mlp.experts.linear_fc1.weight1
1311
- module.decoder.layers.0.mlp.experts.linear_fc2.weight12
1312
- module.decoder.layers.1.mlp.experts.linear_fc1.weight2
1313
- module.decoder.layers.1.mlp.experts.linear_fc1.weight0
1314
  module.decoder.layers.0.mlp.experts.linear_fc2.weight15
1315
  module.decoder.layers.0.mlp.experts.linear_fc2.weight13
1316
  module.decoder.layers.0.mlp.experts.linear_fc2.weight11
 
 
 
 
1317
  Params for bucket 64 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1318
  module.decoder.layers.0.mlp.experts.linear_fc2.weight10
1319
  module.decoder.layers.0.mlp.experts.linear_fc2.weight8
1320
  module.decoder.layers.0.mlp.experts.linear_fc2.weight6
1321
  module.decoder.layers.0.mlp.experts.linear_fc2.weight4
1322
  module.decoder.layers.0.mlp.experts.linear_fc2.weight2
1323
  module.decoder.layers.0.mlp.experts.linear_fc2.weight0
1324
- module.decoder.layers.0.mlp.experts.linear_fc2.weight9
1325
- module.decoder.layers.0.mlp.experts.linear_fc2.weight7
1326
- module.decoder.layers.0.mlp.experts.linear_fc2.weight5
1327
- module.decoder.layers.0.mlp.experts.linear_fc2.weight3
1328
- module.decoder.layers.0.mlp.experts.linear_fc2.weight1
1329
  Params for bucket 65 (44040192 elements, 44040192 padded size):
1330
- module.decoder.layers.0.mlp.experts.linear_fc1.weight15
1331
- module.decoder.layers.0.mlp.experts.linear_fc1.weight13
1332
- module.decoder.layers.0.mlp.experts.linear_fc1.weight11
1333
- module.decoder.layers.0.mlp.experts.linear_fc1.weight12
1334
  module.decoder.layers.0.mlp.experts.linear_fc1.weight14
 
1335
  module.decoder.layers.0.mlp.experts.linear_fc1.weight10
 
 
 
1336
  Params for bucket 66 (44040192 elements, 44040192 padded size):
 
 
 
1337
  module.decoder.layers.0.mlp.experts.linear_fc1.weight9
1338
  module.decoder.layers.0.mlp.experts.linear_fc1.weight5
1339
  module.decoder.layers.0.mlp.experts.linear_fc1.weight7
1340
- module.decoder.layers.0.mlp.experts.linear_fc1.weight6
1341
- module.decoder.layers.0.mlp.experts.linear_fc1.weight8
1342
- module.decoder.layers.0.mlp.experts.linear_fc1.weight4
1343
  Params for bucket 67 (29360128 elements, 29360128 padded size):
1344
  module.decoder.layers.0.mlp.experts.linear_fc1.weight3
 
1345
  module.decoder.layers.0.mlp.experts.linear_fc1.weight2
1346
  module.decoder.layers.0.mlp.experts.linear_fc1.weight1
1347
- module.decoder.layers.0.mlp.experts.linear_fc1.weight0
1348
- INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x78f6c1f6a6f0>, config_logger_dir='')
1349
  INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1350
- [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-17 03:39:24
1351
  > building train, validation, and test datasets ...
1352
  > datasets target sizes (minimum size):
1353
  train: 160
@@ -1355,7 +1355,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1355
  test: 0
1356
  INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
1357
  > building train, validation, and test datasets for GPT ...
1358
- INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x78f6c2bda780>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
1359
  INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
1360
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
1361
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
@@ -1368,14 +1368,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 68684a515e11
1368
  INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy
1369
  INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
1370
  > finished creating GPT datasets ...
1371
- [after dataloaders are built] datetime: 2026-09-17 03:39:24
1372
  done with setup ...
1373
  (min, max) time across ranks (ms):
1374
- model-and-optimizer-setup ......................: (4148.18, 4158.05)
1375
- train/valid/test-data-iterators-setup ..........: (108.26, 139.90)
1376
  training ...
1377
  Overwriting rerun_state_machine.current_iteration from -1 to 0...
1378
- [before the start of training step] datetime: 2026-09-17 03:39:24
1379
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1380
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1381
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
@@ -1390,6 +1390,10 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1390
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1391
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1392
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
 
 
 
 
1393
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1394
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1395
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
@@ -1404,10 +1408,6 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1404
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1405
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1406
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1407
- /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1408
- warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1409
- /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1410
- warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1411
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1412
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1413
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
@@ -1426,8 +1426,6 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1426
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1427
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1428
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1429
- /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1430
- warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1431
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1432
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1433
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
@@ -1436,6 +1434,12 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1436
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1437
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1438
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
 
 
 
 
 
 
1439
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1440
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1441
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
@@ -1452,7 +1456,116 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1452
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1453
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1454
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1455
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1456
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1457
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1458
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  memory_dump=none snapshots=/workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/memory_snapshots/snapshot.pickle
7
  repo=/workspace/Megatron-UltraEP @ 03d57ef7
8
  ==============================================================
9
+ W0921 05:37:36.882000 71874 torch/distributed/run.py:766]
10
+ W0921 05:37:36.882000 71874 torch/distributed/run.py:766] *****************************************
11
+ W0921 05:37:36.882000 71874 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
+ W0921 05:37:36.882000 71874 torch/distributed/run.py:766] *****************************************
13
  using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
14
  WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
15
  Number of virtual stages per pipeline stage: None
 
574
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
575
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
576
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
577
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
578
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
579
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
580
  /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
581
  warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
582
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
 
583
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
  [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
 
585
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
 
586
  [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
  [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
  [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
  [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
  [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
596
  [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
597
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
598
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
599
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
600
  [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
 
605
  > initialized tensor model parallel with size 1
606
  > initialized pipeline model parallel with size 1
607
  > setting random seeds to 1234 ...
608
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
609
+ [rank3]:[W921 05:37:50.095937267 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
+ [rank1]:[W921 05:37:50.100592308 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank4]:[W921 05:37:50.101084131 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank5]:[W921 05:37:50.101408961 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ [rank2]:[W921 05:37:50.101754838 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
614
+ [rank6]:[W921 05:37:50.101776777 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
615
+ [rank7]:[W921 05:37:50.106206547 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
616
  > compiling dataset index builder ...
617
  make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
618
  make: Nothing to be done for 'default'.
619
  make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets'
620
+ >>> done with dataset index builder. Compilation time: 0.436 seconds
621
  WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
622
  > compiling and loading fused kernels ...
623
+ [rank0]:[W921 05:37:50.714007673 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
624
+ >>> done with compiling and loading fused kernels. Compilation time: 4.204 seconds
625
+ time to initialize megatron (seconds): 9.623
626
+ [after megatron is initialized] datetime: 2026-09-21 05:37:57
627
  building GPT model ...
628
  /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
629
  warnings.warn(
 
674
  module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
675
  module.decoder.layers.15.self_attention.linear_proj.weight
676
  Params for bucket 3 (42475520 elements, 42475520 padded size):
 
 
677
  module.decoder.layers.14.mlp.router.weight
 
678
  module.decoder.layers.14.self_attention.linear_proj.weight
679
+ module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
680
+ module.decoder.layers.14.self_attention.linear_qkv.weight
681
+ module.decoder.layers.14.pre_mlp_layernorm.weight
682
  Params for bucket 4 (42475520 elements, 42475520 padded size):
683
  module.decoder.layers.13.mlp.router.weight
 
684
  module.decoder.layers.13.self_attention.linear_proj.weight
685
+ module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
686
  module.decoder.layers.13.self_attention.linear_qkv.weight
687
  module.decoder.layers.13.pre_mlp_layernorm.weight
688
  Params for bucket 5 (42475520 elements, 42475520 padded size):
689
  module.decoder.layers.12.self_attention.linear_qkv.weight
690
  module.decoder.layers.12.pre_mlp_layernorm.weight
691
  module.decoder.layers.12.mlp.router.weight
 
692
  module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
693
+ module.decoder.layers.12.self_attention.linear_proj.weight
694
  Params for bucket 6 (42475520 elements, 42475520 padded size):
 
695
  module.decoder.layers.11.pre_mlp_layernorm.weight
696
+ module.decoder.layers.11.self_attention.linear_qkv.weight
697
  module.decoder.layers.11.mlp.router.weight
698
  module.decoder.layers.11.self_attention.linear_proj.weight
699
  module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
700
  Params for bucket 7 (42475520 elements, 42475520 padded size):
701
+ module.decoder.layers.10.self_attention.linear_qkv.weight
702
+ module.decoder.layers.10.pre_mlp_layernorm.weight
703
  module.decoder.layers.10.mlp.router.weight
704
  module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
705
  module.decoder.layers.10.self_attention.linear_proj.weight
 
 
706
  Params for bucket 8 (42475520 elements, 42475520 padded size):
707
+ module.decoder.layers.9.pre_mlp_layernorm.weight
708
+ module.decoder.layers.9.self_attention.linear_qkv.weight
709
  module.decoder.layers.9.mlp.router.weight
710
  module.decoder.layers.9.self_attention.linear_proj.weight
711
  module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
 
 
712
  Params for bucket 9 (42475520 elements, 42475520 padded size):
713
+ module.decoder.layers.8.self_attention.linear_qkv.weight
714
+ module.decoder.layers.8.pre_mlp_layernorm.weight
715
  module.decoder.layers.8.mlp.router.weight
716
  module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
717
  module.decoder.layers.8.self_attention.linear_proj.weight
 
 
718
  Params for bucket 10 (42475520 elements, 42475520 padded size):
719
+ module.decoder.layers.7.pre_mlp_layernorm.weight
720
+ module.decoder.layers.7.self_attention.linear_qkv.weight
721
  module.decoder.layers.7.mlp.router.weight
722
  module.decoder.layers.7.self_attention.linear_proj.weight
723
  module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
 
 
724
  Params for bucket 11 (42475520 elements, 42475520 padded size):
 
 
725
  module.decoder.layers.6.mlp.router.weight
726
  module.decoder.layers.6.self_attention.linear_proj.weight
727
  module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
728
+ module.decoder.layers.6.pre_mlp_layernorm.weight
729
+ module.decoder.layers.6.self_attention.linear_qkv.weight
730
  Params for bucket 12 (42475520 elements, 42475520 padded size):
731
+ module.decoder.layers.5.mlp.router.weight
732
+ module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
733
  module.decoder.layers.5.self_attention.linear_qkv.weight
734
  module.decoder.layers.5.self_attention.linear_proj.weight
735
  module.decoder.layers.5.pre_mlp_layernorm.weight
 
 
736
  Params for bucket 13 (42475520 elements, 42475520 padded size):
 
 
 
737
  module.decoder.layers.4.self_attention.linear_qkv.weight
738
  module.decoder.layers.4.pre_mlp_layernorm.weight
739
+ module.decoder.layers.4.mlp.router.weight
740
+ module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
741
+ module.decoder.layers.4.self_attention.linear_proj.weight
742
  Params for bucket 14 (42475520 elements, 42475520 padded size):
743
+ module.decoder.layers.3.pre_mlp_layernorm.weight
744
+ module.decoder.layers.3.self_attention.linear_qkv.weight
745
  module.decoder.layers.3.mlp.router.weight
746
  module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
747
  module.decoder.layers.3.self_attention.linear_proj.weight
 
 
748
  Params for bucket 15 (42475520 elements, 42475520 padded size):
749
  module.decoder.layers.2.mlp.router.weight
750
  module.decoder.layers.2.self_attention.linear_proj.weight
 
752
  module.decoder.layers.2.pre_mlp_layernorm.weight
753
  module.decoder.layers.2.self_attention.linear_qkv.weight
754
  Params for bucket 16 (42475520 elements, 42475520 padded size):
 
 
755
  module.decoder.layers.1.mlp.router.weight
 
756
  module.decoder.layers.1.self_attention.linear_proj.weight
757
+ module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
758
+ module.decoder.layers.1.pre_mlp_layernorm.weight
759
+ module.decoder.layers.1.self_attention.linear_qkv.weight
760
  Params for bucket 17 (42475520 elements, 42475520 padded size):
761
  module.decoder.layers.0.mlp.router.weight
762
  module.decoder.layers.0.pre_mlp_layernorm.weight
 
763
  module.decoder.layers.0.self_attention.linear_qkv.weight
764
+ module.decoder.layers.0.self_attention.linear_proj.weight
765
  module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
766
  Params for bucket 18 (206045184 elements, 206045184 padded size):
767
  module.embedding.word_embeddings.weight
768
  INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67
769
  Params for bucket 1 (40370176 elements, 40370176 padded size):
 
 
 
 
 
770
  module.decoder.layers.15.mlp.experts.linear_fc2.weight15
771
  module.decoder.layers.15.mlp.experts.linear_fc2.weight13
772
  module.decoder.layers.15.mlp.experts.linear_fc2.weight11
773
  module.decoder.layers.15.mlp.experts.linear_fc2.weight9
774
  module.decoder.layers.15.mlp.experts.linear_fc2.weight7
775
  module.decoder.layers.15.mlp.experts.linear_fc2.weight5
776
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight12
777
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight14
778
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight10
779
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight8
780
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight6
781
  Params for bucket 2 (40370176 elements, 40370176 padded size):
 
 
 
 
 
782
  module.decoder.layers.15.mlp.experts.linear_fc2.weight3
783
  module.decoder.layers.15.mlp.experts.linear_fc2.weight1
784
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight15
785
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight13
786
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight4
787
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight0
788
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight2
789
  module.decoder.layers.15.mlp.experts.linear_fc1.weight14
790
  Params for bucket 3 (44040192 elements, 44040192 padded size):
791
  module.decoder.layers.15.mlp.experts.linear_fc1.weight11
 
799
  module.decoder.layers.15.mlp.experts.linear_fc1.weight3
800
  module.decoder.layers.15.mlp.experts.linear_fc1.weight1
801
  module.decoder.layers.15.mlp.experts.linear_fc1.weight6
 
802
  module.decoder.layers.15.mlp.experts.linear_fc1.weight4
803
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight2
804
  Params for bucket 5 (40370176 elements, 40370176 padded size):
 
805
  module.decoder.layers.14.mlp.experts.linear_fc2.weight12
806
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight14
807
  module.decoder.layers.14.mlp.experts.linear_fc2.weight10
808
  module.decoder.layers.14.mlp.experts.linear_fc2.weight8
809
  module.decoder.layers.15.mlp.experts.linear_fc1.weight0
 
810
  module.decoder.layers.14.mlp.experts.linear_fc2.weight13
811
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight15
812
  module.decoder.layers.14.mlp.experts.linear_fc2.weight11
813
  module.decoder.layers.14.mlp.experts.linear_fc2.weight9
814
  module.decoder.layers.14.mlp.experts.linear_fc2.weight7
 
818
  module.decoder.layers.14.mlp.experts.linear_fc2.weight2
819
  module.decoder.layers.14.mlp.experts.linear_fc2.weight0
820
  module.decoder.layers.14.mlp.experts.linear_fc1.weight15
 
821
  module.decoder.layers.14.mlp.experts.linear_fc2.weight3
822
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight5
823
  module.decoder.layers.14.mlp.experts.linear_fc2.weight1
824
  module.decoder.layers.14.mlp.experts.linear_fc1.weight14
825
  Params for bucket 7 (44040192 elements, 44040192 padded size):
826
  module.decoder.layers.14.mlp.experts.linear_fc1.weight13
 
827
  module.decoder.layers.14.mlp.experts.linear_fc1.weight9
828
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight11
829
  module.decoder.layers.14.mlp.experts.linear_fc1.weight10
830
  module.decoder.layers.14.mlp.experts.linear_fc1.weight12
831
  module.decoder.layers.14.mlp.experts.linear_fc1.weight8
832
  Params for bucket 8 (44040192 elements, 44040192 padded size):
833
  module.decoder.layers.14.mlp.experts.linear_fc1.weight7
 
834
  module.decoder.layers.14.mlp.experts.linear_fc1.weight3
835
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight5
836
  module.decoder.layers.14.mlp.experts.linear_fc1.weight6
837
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight4
838
  module.decoder.layers.14.mlp.experts.linear_fc1.weight2
839
  Params for bucket 9 (40370176 elements, 40370176 padded size):
840
  module.decoder.layers.14.mlp.experts.linear_fc1.weight1
 
 
 
 
841
  module.decoder.layers.13.mlp.experts.linear_fc2.weight15
842
  module.decoder.layers.13.mlp.experts.linear_fc2.weight13
843
  module.decoder.layers.13.mlp.experts.linear_fc2.weight11
844
  module.decoder.layers.13.mlp.experts.linear_fc2.weight9
845
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight0
846
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight14
847
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight12
848
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight10
849
  Params for bucket 10 (40370176 elements, 40370176 padded size):
850
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight7
851
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight5
852
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight3
853
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight1
854
  module.decoder.layers.13.mlp.experts.linear_fc2.weight8
855
  module.decoder.layers.13.mlp.experts.linear_fc2.weight4
856
  module.decoder.layers.13.mlp.experts.linear_fc2.weight6
857
  module.decoder.layers.13.mlp.experts.linear_fc2.weight2
858
  module.decoder.layers.13.mlp.experts.linear_fc2.weight0
 
 
 
 
859
  module.decoder.layers.13.mlp.experts.linear_fc1.weight15
860
  Params for bucket 11 (44040192 elements, 44040192 padded size):
861
  module.decoder.layers.13.mlp.experts.linear_fc1.weight14
862
  module.decoder.layers.13.mlp.experts.linear_fc1.weight10
863
  module.decoder.layers.13.mlp.experts.linear_fc1.weight12
 
864
  module.decoder.layers.13.mlp.experts.linear_fc1.weight13
865
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight11
866
  module.decoder.layers.13.mlp.experts.linear_fc1.weight9
867
  Params for bucket 12 (44040192 elements, 44040192 padded size):
868
  module.decoder.layers.13.mlp.experts.linear_fc1.weight8
869
  module.decoder.layers.13.mlp.experts.linear_fc1.weight4
870
  module.decoder.layers.13.mlp.experts.linear_fc1.weight6
 
871
  module.decoder.layers.13.mlp.experts.linear_fc1.weight7
872
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight5
873
  module.decoder.layers.13.mlp.experts.linear_fc1.weight3
874
  Params for bucket 13 (40370176 elements, 40370176 padded size):
875
  module.decoder.layers.13.mlp.experts.linear_fc1.weight2
 
876
  module.decoder.layers.13.mlp.experts.linear_fc1.weight0
877
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight15
878
  module.decoder.layers.12.mlp.experts.linear_fc2.weight13
879
  module.decoder.layers.12.mlp.experts.linear_fc2.weight11
880
  module.decoder.layers.13.mlp.experts.linear_fc1.weight1
881
  module.decoder.layers.12.mlp.experts.linear_fc2.weight14
882
  module.decoder.layers.12.mlp.experts.linear_fc2.weight12
883
  Params for bucket 14 (40370176 elements, 40370176 padded size):
 
884
  module.decoder.layers.12.mlp.experts.linear_fc2.weight7
885
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight9
886
  module.decoder.layers.12.mlp.experts.linear_fc2.weight5
887
  module.decoder.layers.12.mlp.experts.linear_fc2.weight3
888
  module.decoder.layers.12.mlp.experts.linear_fc2.weight1
 
893
  module.decoder.layers.12.mlp.experts.linear_fc2.weight2
894
  module.decoder.layers.12.mlp.experts.linear_fc2.weight0
895
  Params for bucket 15 (44040192 elements, 44040192 padded size):
 
 
 
896
  module.decoder.layers.12.mlp.experts.linear_fc1.weight15
897
  module.decoder.layers.12.mlp.experts.linear_fc1.weight11
898
  module.decoder.layers.12.mlp.experts.linear_fc1.weight13
899
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight14
900
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight12
901
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight10
902
  Params for bucket 16 (44040192 elements, 44040192 padded size):
 
 
 
903
  module.decoder.layers.12.mlp.experts.linear_fc1.weight9
904
  module.decoder.layers.12.mlp.experts.linear_fc1.weight7
905
  module.decoder.layers.12.mlp.experts.linear_fc1.weight5
906
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight8
907
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight6
908
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight4
909
  Params for bucket 17 (40370176 elements, 40370176 padded size):
 
 
 
910
  module.decoder.layers.12.mlp.experts.linear_fc1.weight3
 
911
  module.decoder.layers.12.mlp.experts.linear_fc1.weight1
912
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight14
913
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight2
914
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight15
915
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight0
916
  module.decoder.layers.11.mlp.experts.linear_fc2.weight13
917
  Params for bucket 18 (40370176 elements, 40370176 padded size):
918
  module.decoder.layers.11.mlp.experts.linear_fc2.weight12
 
919
  module.decoder.layers.11.mlp.experts.linear_fc2.weight8
920
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight10
921
  module.decoder.layers.11.mlp.experts.linear_fc2.weight6
922
  module.decoder.layers.11.mlp.experts.linear_fc2.weight4
923
  module.decoder.layers.11.mlp.experts.linear_fc2.weight2
 
924
  module.decoder.layers.11.mlp.experts.linear_fc2.weight11
925
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight9
926
  module.decoder.layers.11.mlp.experts.linear_fc2.weight7
927
  module.decoder.layers.11.mlp.experts.linear_fc2.weight5
928
  module.decoder.layers.11.mlp.experts.linear_fc2.weight3
929
  Params for bucket 19 (44040192 elements, 44040192 padded size):
930
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight15
931
  module.decoder.layers.11.mlp.experts.linear_fc1.weight13
932
  module.decoder.layers.11.mlp.experts.linear_fc2.weight0
 
933
  module.decoder.layers.11.mlp.experts.linear_fc1.weight11
934
  module.decoder.layers.11.mlp.experts.linear_fc2.weight1
935
  module.decoder.layers.11.mlp.experts.linear_fc1.weight14
936
  module.decoder.layers.11.mlp.experts.linear_fc1.weight12
937
  Params for bucket 20 (44040192 elements, 44040192 padded size):
 
938
  module.decoder.layers.11.mlp.experts.linear_fc1.weight9
939
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight7
940
  module.decoder.layers.11.mlp.experts.linear_fc1.weight5
941
  module.decoder.layers.11.mlp.experts.linear_fc1.weight10
 
942
  module.decoder.layers.11.mlp.experts.linear_fc1.weight6
943
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight8
944
  Params for bucket 21 (40370176 elements, 40370176 padded size):
945
  module.decoder.layers.11.mlp.experts.linear_fc1.weight3
946
  module.decoder.layers.11.mlp.experts.linear_fc1.weight1
 
947
  module.decoder.layers.11.mlp.experts.linear_fc1.weight4
 
948
  module.decoder.layers.11.mlp.experts.linear_fc1.weight2
949
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight0
950
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight15
951
  Params for bucket 22 (40370176 elements, 40370176 padded size):
 
 
 
 
 
952
  module.decoder.layers.10.mlp.experts.linear_fc2.weight14
953
  module.decoder.layers.10.mlp.experts.linear_fc2.weight10
954
  module.decoder.layers.10.mlp.experts.linear_fc2.weight12
955
  module.decoder.layers.10.mlp.experts.linear_fc2.weight8
956
  module.decoder.layers.10.mlp.experts.linear_fc2.weight6
957
  module.decoder.layers.10.mlp.experts.linear_fc2.weight4
958
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight13
959
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight11
960
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight9
961
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight7
962
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight5
963
  Params for bucket 23 (44040192 elements, 44040192 padded size):
 
 
 
 
964
  module.decoder.layers.10.mlp.experts.linear_fc2.weight0
965
  module.decoder.layers.10.mlp.experts.linear_fc1.weight15
966
  module.decoder.layers.10.mlp.experts.linear_fc2.weight2
967
  module.decoder.layers.10.mlp.experts.linear_fc1.weight13
968
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight3
969
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight1
970
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight14
971
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight12
972
  Params for bucket 24 (44040192 elements, 44040192 padded size):
 
 
 
973
  module.decoder.layers.10.mlp.experts.linear_fc1.weight11
974
  module.decoder.layers.10.mlp.experts.linear_fc1.weight7
975
  module.decoder.layers.10.mlp.experts.linear_fc1.weight9
976
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight8
977
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight10
978
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight6
979
  Params for bucket 25 (44040192 elements, 44040192 padded size):
 
 
 
980
  module.decoder.layers.10.mlp.experts.linear_fc1.weight5
981
  module.decoder.layers.10.mlp.experts.linear_fc1.weight1
982
  module.decoder.layers.10.mlp.experts.linear_fc1.weight3
983
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight4
984
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight2
985
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight0
986
  Params for bucket 26 (40370176 elements, 40370176 padded size):
987
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight12
988
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight14
989
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight10
990
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight8
991
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight6
992
  module.decoder.layers.9.mlp.experts.linear_fc2.weight15
993
  module.decoder.layers.9.mlp.experts.linear_fc2.weight11
994
  module.decoder.layers.9.mlp.experts.linear_fc2.weight13
995
  module.decoder.layers.9.mlp.experts.linear_fc2.weight9
996
  module.decoder.layers.9.mlp.experts.linear_fc2.weight7
997
  module.decoder.layers.9.mlp.experts.linear_fc2.weight5
 
 
 
 
 
998
  Params for bucket 27 (40370176 elements, 40370176 padded size):
 
 
 
999
  module.decoder.layers.9.mlp.experts.linear_fc2.weight4
1000
  module.decoder.layers.9.mlp.experts.linear_fc2.weight2
1001
  module.decoder.layers.9.mlp.experts.linear_fc2.weight0
1002
  module.decoder.layers.9.mlp.experts.linear_fc1.weight15
1003
  module.decoder.layers.9.mlp.experts.linear_fc1.weight13
1004
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight3
1005
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight1
1006
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight14
1007
  Params for bucket 28 (44040192 elements, 44040192 padded size):
 
 
 
1008
  module.decoder.layers.9.mlp.experts.linear_fc1.weight9
1009
  module.decoder.layers.9.mlp.experts.linear_fc1.weight11
1010
  module.decoder.layers.9.mlp.experts.linear_fc1.weight7
1011
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight12
1012
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight8
1013
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight10
1014
  Params for bucket 29 (44040192 elements, 44040192 padded size):
 
 
 
1015
  module.decoder.layers.9.mlp.experts.linear_fc1.weight3
1016
  module.decoder.layers.9.mlp.experts.linear_fc1.weight5
1017
  module.decoder.layers.9.mlp.experts.linear_fc1.weight1
1018
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight6
1019
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight2
1020
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight4
1021
  Params for bucket 30 (40370176 elements, 40370176 padded size):
1022
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight14
1023
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight12
1024
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight10
1025
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight8
1026
  module.decoder.layers.9.mlp.experts.linear_fc1.weight0
1027
  module.decoder.layers.8.mlp.experts.linear_fc2.weight13
1028
  module.decoder.layers.8.mlp.experts.linear_fc2.weight15
1029
  module.decoder.layers.8.mlp.experts.linear_fc2.weight11
1030
  module.decoder.layers.8.mlp.experts.linear_fc2.weight9
1031
  module.decoder.layers.8.mlp.experts.linear_fc2.weight7
 
 
 
 
1032
  Params for bucket 31 (40370176 elements, 40370176 padded size):
 
 
 
 
1033
  module.decoder.layers.8.mlp.experts.linear_fc2.weight6
1034
  module.decoder.layers.8.mlp.experts.linear_fc2.weight4
1035
  module.decoder.layers.8.mlp.experts.linear_fc2.weight2
1036
  module.decoder.layers.8.mlp.experts.linear_fc2.weight0
1037
  module.decoder.layers.8.mlp.experts.linear_fc1.weight15
1038
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight3
1039
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight5
1040
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight1
1041
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight14
1042
  Params for bucket 32 (44040192 elements, 44040192 padded size):
 
 
 
1043
  module.decoder.layers.8.mlp.experts.linear_fc1.weight13
1044
  module.decoder.layers.8.mlp.experts.linear_fc1.weight11
1045
  module.decoder.layers.8.mlp.experts.linear_fc1.weight9
1046
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight12
1047
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight10
1048
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight8
1049
  Params for bucket 33 (44040192 elements, 44040192 padded size):
1050
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight7
1051
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight5
1052
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight3
1053
  module.decoder.layers.8.mlp.experts.linear_fc1.weight4
1054
  module.decoder.layers.8.mlp.experts.linear_fc1.weight6
1055
  module.decoder.layers.8.mlp.experts.linear_fc1.weight2
 
 
 
1056
  Params for bucket 34 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1057
  module.decoder.layers.8.mlp.experts.linear_fc1.weight1
1058
  module.decoder.layers.7.mlp.experts.linear_fc2.weight14
1059
  module.decoder.layers.7.mlp.experts.linear_fc2.weight12
1060
  module.decoder.layers.7.mlp.experts.linear_fc2.weight10
1061
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight15
1062
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight13
1063
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight0
1064
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight11
1065
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight9
1066
+ Params for bucket 35 (40370176 elements, 40370176 padded size):
1067
  module.decoder.layers.7.mlp.experts.linear_fc2.weight8
 
1068
  module.decoder.layers.7.mlp.experts.linear_fc2.weight6
1069
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight4
1070
  module.decoder.layers.7.mlp.experts.linear_fc2.weight2
1071
  module.decoder.layers.7.mlp.experts.linear_fc2.weight0
1072
  module.decoder.layers.7.mlp.experts.linear_fc1.weight15
1073
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight5
1074
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight7
1075
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight3
1076
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight1
1077
  Params for bucket 36 (44040192 elements, 44040192 padded size):
 
 
 
1078
  module.decoder.layers.7.mlp.experts.linear_fc1.weight13
1079
  module.decoder.layers.7.mlp.experts.linear_fc1.weight11
1080
  module.decoder.layers.7.mlp.experts.linear_fc1.weight9
1081
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight14
1082
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight12
1083
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight10
1084
  Params for bucket 37 (44040192 elements, 44040192 padded size):
 
 
 
1085
  module.decoder.layers.7.mlp.experts.linear_fc1.weight7
1086
  module.decoder.layers.7.mlp.experts.linear_fc1.weight5
1087
  module.decoder.layers.7.mlp.experts.linear_fc1.weight3
1088
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight8
1089
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight6
1090
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight4
1091
  Params for bucket 38 (40370176 elements, 40370176 padded size):
1092
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight14
1093
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight1
1094
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight12
1095
  module.decoder.layers.7.mlp.experts.linear_fc1.weight2
 
1096
  module.decoder.layers.6.mlp.experts.linear_fc2.weight15
1097
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight0
1098
  module.decoder.layers.6.mlp.experts.linear_fc2.weight13
1099
  module.decoder.layers.6.mlp.experts.linear_fc2.weight11
 
 
 
1100
  Params for bucket 39 (40370176 elements, 40370176 padded size):
 
 
 
 
 
1101
  module.decoder.layers.6.mlp.experts.linear_fc2.weight10
1102
  module.decoder.layers.6.mlp.experts.linear_fc2.weight6
1103
  module.decoder.layers.6.mlp.experts.linear_fc2.weight8
1104
  module.decoder.layers.6.mlp.experts.linear_fc2.weight4
1105
  module.decoder.layers.6.mlp.experts.linear_fc2.weight2
1106
  module.decoder.layers.6.mlp.experts.linear_fc2.weight0
1107
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight7
1108
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight9
1109
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight5
1110
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight3
1111
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight1
1112
  Params for bucket 40 (44040192 elements, 44040192 padded size):
 
 
 
 
1113
  module.decoder.layers.6.mlp.experts.linear_fc1.weight14
1114
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight12
1115
  module.decoder.layers.6.mlp.experts.linear_fc1.weight10
1116
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight15
1117
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight11
1118
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight13
1119
  Params for bucket 41 (44040192 elements, 44040192 padded size):
 
 
 
1120
  module.decoder.layers.6.mlp.experts.linear_fc1.weight6
1121
  module.decoder.layers.6.mlp.experts.linear_fc1.weight8
1122
  module.decoder.layers.6.mlp.experts.linear_fc1.weight4
1123
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight9
1124
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight7
1125
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight5
1126
  Params for bucket 42 (40370176 elements, 40370176 padded size):
 
 
 
1127
  module.decoder.layers.6.mlp.experts.linear_fc1.weight0
1128
  module.decoder.layers.5.mlp.experts.linear_fc2.weight15
1129
  module.decoder.layers.6.mlp.experts.linear_fc1.weight2
1130
  module.decoder.layers.5.mlp.experts.linear_fc2.weight13
1131
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight3
1132
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight1
1133
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight14
1134
  Params for bucket 43 (40370176 elements, 40370176 padded size):
 
 
 
 
 
 
1135
  module.decoder.layers.5.mlp.experts.linear_fc2.weight11
1136
  module.decoder.layers.5.mlp.experts.linear_fc2.weight9
1137
  module.decoder.layers.5.mlp.experts.linear_fc2.weight7
1138
  module.decoder.layers.5.mlp.experts.linear_fc2.weight5
1139
  module.decoder.layers.5.mlp.experts.linear_fc2.weight3
1140
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight12
1141
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight8
1142
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight10
1143
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight6
1144
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight4
1145
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight2
1146
  Params for bucket 44 (44040192 elements, 44040192 padded size):
 
 
 
 
1147
  module.decoder.layers.5.mlp.experts.linear_fc2.weight1
1148
  module.decoder.layers.5.mlp.experts.linear_fc1.weight14
1149
  module.decoder.layers.5.mlp.experts.linear_fc1.weight12
1150
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight13
1151
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight15
1152
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight0
1153
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight11
1154
  Params for bucket 45 (44040192 elements, 44040192 padded size):
1155
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight10
1156
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight8
1157
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight6
1158
  module.decoder.layers.5.mlp.experts.linear_fc1.weight9
1159
  module.decoder.layers.5.mlp.experts.linear_fc1.weight7
1160
  module.decoder.layers.5.mlp.experts.linear_fc1.weight5
 
 
 
1161
  Params for bucket 46 (40370176 elements, 40370176 padded size):
 
 
1162
  module.decoder.layers.5.mlp.experts.linear_fc1.weight4
1163
  module.decoder.layers.5.mlp.experts.linear_fc1.weight2
1164
  module.decoder.layers.5.mlp.experts.linear_fc1.weight0
1165
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight1
1166
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight3
1167
  module.decoder.layers.4.mlp.experts.linear_fc2.weight15
1168
  Params for bucket 47 (40370176 elements, 40370176 padded size):
1169
  module.decoder.layers.4.mlp.experts.linear_fc2.weight14
 
1170
  module.decoder.layers.4.mlp.experts.linear_fc2.weight10
1171
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight12
1172
  module.decoder.layers.4.mlp.experts.linear_fc2.weight8
1173
  module.decoder.layers.4.mlp.experts.linear_fc2.weight6
1174
  module.decoder.layers.4.mlp.experts.linear_fc2.weight4
 
1178
  module.decoder.layers.4.mlp.experts.linear_fc2.weight7
1179
  module.decoder.layers.4.mlp.experts.linear_fc2.weight5
1180
  Params for bucket 48 (44040192 elements, 44040192 padded size):
 
1181
  module.decoder.layers.4.mlp.experts.linear_fc2.weight0
1182
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight2
 
 
1183
  module.decoder.layers.4.mlp.experts.linear_fc1.weight15
 
1184
  module.decoder.layers.4.mlp.experts.linear_fc1.weight13
1185
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight3
1186
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight1
1187
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight14
1188
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight12
1189
  Params for bucket 49 (44040192 elements, 44040192 padded size):
 
 
 
1190
  module.decoder.layers.4.mlp.experts.linear_fc1.weight11
1191
  module.decoder.layers.4.mlp.experts.linear_fc1.weight9
1192
  module.decoder.layers.4.mlp.experts.linear_fc1.weight7
1193
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight10
1194
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight8
1195
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight6
1196
  Params for bucket 50 (44040192 elements, 44040192 padded size):
 
 
 
1197
  module.decoder.layers.4.mlp.experts.linear_fc1.weight5
1198
  module.decoder.layers.4.mlp.experts.linear_fc1.weight1
1199
  module.decoder.layers.4.mlp.experts.linear_fc1.weight3
1200
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight4
1201
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight2
1202
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight0
1203
  Params for bucket 51 (40370176 elements, 40370176 padded size):
1204
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight12
1205
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight14
1206
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight10
1207
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight8
1208
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight6
1209
  module.decoder.layers.3.mlp.experts.linear_fc2.weight15
1210
  module.decoder.layers.3.mlp.experts.linear_fc2.weight11
1211
  module.decoder.layers.3.mlp.experts.linear_fc2.weight13
1212
  module.decoder.layers.3.mlp.experts.linear_fc2.weight9
1213
  module.decoder.layers.3.mlp.experts.linear_fc2.weight7
1214
  module.decoder.layers.3.mlp.experts.linear_fc2.weight5
 
 
 
 
 
1215
  Params for bucket 52 (40370176 elements, 40370176 padded size):
 
 
 
1216
  module.decoder.layers.3.mlp.experts.linear_fc2.weight4
 
1217
  module.decoder.layers.3.mlp.experts.linear_fc2.weight2
1218
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight0
1219
  module.decoder.layers.3.mlp.experts.linear_fc1.weight15
1220
  module.decoder.layers.3.mlp.experts.linear_fc1.weight13
1221
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight3
1222
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight1
1223
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight14
1224
  Params for bucket 53 (44040192 elements, 44040192 padded size):
 
 
 
1225
  module.decoder.layers.3.mlp.experts.linear_fc1.weight11
1226
  module.decoder.layers.3.mlp.experts.linear_fc1.weight9
1227
  module.decoder.layers.3.mlp.experts.linear_fc1.weight7
1228
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight12
1229
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight10
1230
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight8
1231
  Params for bucket 54 (44040192 elements, 44040192 padded size):
 
 
 
1232
  module.decoder.layers.3.mlp.experts.linear_fc1.weight5
1233
  module.decoder.layers.3.mlp.experts.linear_fc1.weight3
1234
  module.decoder.layers.3.mlp.experts.linear_fc1.weight1
1235
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight6
1236
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight2
1237
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight4
1238
  Params for bucket 55 (40370176 elements, 40370176 padded size):
1239
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight14
1240
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight12
1241
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight10
1242
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight8
1243
  module.decoder.layers.3.mlp.experts.linear_fc1.weight0
 
1244
  module.decoder.layers.2.mlp.experts.linear_fc2.weight13
1245
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight15
1246
  module.decoder.layers.2.mlp.experts.linear_fc2.weight11
1247
  module.decoder.layers.2.mlp.experts.linear_fc2.weight9
1248
  module.decoder.layers.2.mlp.experts.linear_fc2.weight7
 
 
 
 
1249
  Params for bucket 56 (40370176 elements, 40370176 padded size):
 
 
 
 
1250
  module.decoder.layers.2.mlp.experts.linear_fc2.weight6
1251
  module.decoder.layers.2.mlp.experts.linear_fc2.weight4
1252
  module.decoder.layers.2.mlp.experts.linear_fc2.weight2
1253
  module.decoder.layers.2.mlp.experts.linear_fc2.weight0
1254
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight14
1255
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight5
1256
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight3
1257
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight1
1258
  module.decoder.layers.2.mlp.experts.linear_fc1.weight15
1259
  Params for bucket 57 (44040192 elements, 44040192 padded size):
1260
  module.decoder.layers.2.mlp.experts.linear_fc1.weight12
 
1296
  module.decoder.layers.1.mlp.experts.linear_fc1.weight11
1297
  module.decoder.layers.1.mlp.experts.linear_fc1.weight9
1298
  module.decoder.layers.1.mlp.experts.linear_fc1.weight14
 
1299
  module.decoder.layers.1.mlp.experts.linear_fc1.weight12
1300
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
1301
  Params for bucket 62 (44040192 elements, 44040192 padded size):
1302
  module.decoder.layers.1.mlp.experts.linear_fc1.weight7
1303
  module.decoder.layers.1.mlp.experts.linear_fc1.weight5
 
1306
  module.decoder.layers.1.mlp.experts.linear_fc1.weight6
1307
  module.decoder.layers.1.mlp.experts.linear_fc1.weight4
1308
  Params for bucket 63 (40370176 elements, 40370176 padded size):
 
1309
  module.decoder.layers.1.mlp.experts.linear_fc1.weight1
 
 
 
1310
  module.decoder.layers.0.mlp.experts.linear_fc2.weight15
1311
  module.decoder.layers.0.mlp.experts.linear_fc2.weight13
1312
  module.decoder.layers.0.mlp.experts.linear_fc2.weight11
1313
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
1314
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
1315
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight14
1316
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight12
1317
  Params for bucket 64 (40370176 elements, 40370176 padded size):
1318
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight7
1319
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight9
1320
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight5
1321
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight3
1322
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight1
1323
  module.decoder.layers.0.mlp.experts.linear_fc2.weight10
1324
  module.decoder.layers.0.mlp.experts.linear_fc2.weight8
1325
  module.decoder.layers.0.mlp.experts.linear_fc2.weight6
1326
  module.decoder.layers.0.mlp.experts.linear_fc2.weight4
1327
  module.decoder.layers.0.mlp.experts.linear_fc2.weight2
1328
  module.decoder.layers.0.mlp.experts.linear_fc2.weight0
 
 
 
 
 
1329
  Params for bucket 65 (44040192 elements, 44040192 padded size):
 
 
 
 
1330
  module.decoder.layers.0.mlp.experts.linear_fc1.weight14
1331
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight12
1332
  module.decoder.layers.0.mlp.experts.linear_fc1.weight10
1333
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight15
1334
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight11
1335
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight13
1336
  Params for bucket 66 (44040192 elements, 44040192 padded size):
1337
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight8
1338
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight6
1339
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight4
1340
  module.decoder.layers.0.mlp.experts.linear_fc1.weight9
1341
  module.decoder.layers.0.mlp.experts.linear_fc1.weight5
1342
  module.decoder.layers.0.mlp.experts.linear_fc1.weight7
 
 
 
1343
  Params for bucket 67 (29360128 elements, 29360128 padded size):
1344
  module.decoder.layers.0.mlp.experts.linear_fc1.weight3
1345
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight0
1346
  module.decoder.layers.0.mlp.experts.linear_fc1.weight2
1347
  module.decoder.layers.0.mlp.experts.linear_fc1.weight1
1348
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x731b4ebd4ec0>, config_logger_dir='')
 
1349
  INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1350
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-09-21 05:38:01
1351
  > building train, validation, and test datasets ...
1352
  > datasets target sizes (minimum size):
1353
  train: 160
 
1355
  test: 0
1356
  INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
1357
  > building train, validation, and test datasets for GPT ...
1358
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x731b42cc7440>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
1359
  INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
1360
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
1361
  INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
 
1368
  INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy
1369
  INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
1370
  > finished creating GPT datasets ...
1371
+ [after dataloaders are built] datetime: 2026-09-21 05:38:01
1372
  done with setup ...
1373
  (min, max) time across ranks (ms):
1374
+ model-and-optimizer-setup ......................: (3774.00, 3790.63)
1375
+ train/valid/test-data-iterators-setup ..........: (110.58, 130.19)
1376
  training ...
1377
  Overwriting rerun_state_machine.current_iteration from -1 to 0...
1378
+ [before the start of training step] datetime: 2026-09-21 05:38:01
1379
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1380
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1381
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
 
1390
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1391
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1392
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1393
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1394
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1395
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1396
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1397
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1398
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1399
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
 
1408
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1409
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1410
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
 
 
 
 
1411
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1412
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1413
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
 
1426
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1427
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1428
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
 
 
1429
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1430
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1431
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
 
1434
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1435
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1436
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1437
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1438
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1439
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1440
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1441
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1442
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1443
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1444
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1445
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
 
1456
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1457
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1458
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1459
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1460
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1461
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1462
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1463
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1464
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1465
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1466
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1467
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1468
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1469
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1470
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1471
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1472
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1473
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1474
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1475
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1476
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1477
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1478
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1479
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1480
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1481
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1482
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1483
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1484
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1485
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1486
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1487
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1488
  warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1489
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future.
1490
+ warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.")
1491
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1492
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1493
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1494
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1495
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1496
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1497
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1498
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1499
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1500
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1501
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1502
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1503
+ /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1504
+ warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1505
  /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future.
1506
  warnings.warn("The use_host_meta is deprecated, it will be removed in the future.")
1507
+ [2026-09-21 05:38:33] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 32018.7 | throughput per GPU (TFLOP/s/GPU): 2.8 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102382E+01 | loss scale: 1.0 | grad norm: 46.215 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1508
+ Number of parameters in transformer block in billions: 23.22
1509
+ Number of parameters in embedding layers in billions: 0.41
1510
+ Total number of parameters in billions: 23.63
1511
+ Number of parameters in most loaded shard in billions: 23.6320
1512
+ Theoretical memory footprints: weight and optimizer=169029.43 MB
1513
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 58106.212890625 | max allocated: 65265.369140625 | reserved: 71948.0 | max reserved: 71948.0
1514
+ [2026-09-21 05:38:34] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 841.5 | throughput per GPU (TFLOP/s/GPU): 108.1 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101774E+01 | loss scale: 1.0 | grad norm: 47.918 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1515
+ [2026-09-21 05:38:34] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 454.2 | throughput per GPU (TFLOP/s/GPU): 200.2 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.036006E+01 | loss scale: 1.0 | grad norm: 194.327 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1516
+ Capture CUDA graph for training!!!
1517
+ CUDA graph capture done!!!
1518
+ [2026-09-21 05:38:37] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 3142.7 | throughput per GPU (TFLOP/s/GPU): 28.9 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.304710E+01 | loss scale: 1.0 | grad norm: 46.384 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1519
+ [2026-09-21 05:38:37] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 249.1 | throughput per GPU (TFLOP/s/GPU): 365.1 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.035521E+01 | loss scale: 1.0 | grad norm: 18.390 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1520
+ [2026-09-21 05:38:38] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 248.7 | throughput per GPU (TFLOP/s/GPU): 365.7 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.025067E+01 | loss scale: 1.0 | grad norm: 29.367 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1521
+ [2026-09-21 05:38:41] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3376.7 | throughput per GPU (TFLOP/s/GPU): 26.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.934441E+00 | loss scale: 1.0 | grad norm: 8.053 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1522
+ [2026-09-21 05:38:41] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 263.7 | throughput per GPU (TFLOP/s/GPU): 344.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.445292E+00 | loss scale: 1.0 | grad norm: 6.712 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 18048 | min safe capacity: 1.1016 |
1523
+ Capture range started in the application.
1524
+ Capture range ended in the application.
1525
+ Generating '/tmp/nsys-report-5fc0.qdstrm'
1526
+
1527
+
1528
+ Collection stop is not allowed in this state.
1529
+ W0921 05:40:19.568000 71874 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers
1530
+ W0921 05:40:19.569000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71985 closing signal SIGTERM
1531
+ Generated:
1532
+ /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep
1533
+ W0921 05:40:19.571000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71986 closing signal SIGTERM
1534
+ W0921 05:40:19.574000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71987 closing signal SIGTERM
1535
+ W0921 05:40:19.575000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71988 closing signal SIGTERM
1536
+ W0921 05:40:19.579000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71989 closing signal SIGTERM
1537
+ W0921 05:40:19.581000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71990 closing signal SIGTERM
1538
+ W0921 05:40:19.585000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71991 closing signal SIGTERM
1539
+ W0921 05:40:19.587000 71874 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 71992 closing signal SIGTERM
1540
+ Traceback (most recent call last):
1541
+ File "/usr/local/bin/torchrun", line 8, in <module>
1542
+ sys.exit(main())
1543
+ ^^^^^^
1544
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1545
+ return f(*args, **kwargs)
1546
+ ^^^^^^^^^^^^^^^^^^
1547
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1548
+ run(args)
1549
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1550
+ elastic_launch(
1551
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1552
+ return launch_agent(self._config, self._entrypoint, list(args))
1553
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1554
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
1555
+ result = agent.run()
1556
+ ^^^^^^^^^^^
1557
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
1558
+ result = f(*args, **kwargs)
1559
+ ^^^^^^^^^^^^^^^^^^
1560
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
1561
+ result = self._invoke_run(role)
1562
+ ^^^^^^^^^^^^^^^^^^^^^^
1563
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run
1564
+ time.sleep(monitor_interval)
1565
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler
1566
+ raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval)
1567
+ torch.distributed.elastic.multiprocessing.api.SignalException: Process 71874 got signal: 15
1568
+ The target application terminated. One or more process it created re-parented.
1569
+ Waiting for termination of re-parented processes.
1570
+ Use the `--wait` option to modify this behavior.
1571
+ There are no active sessions.
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep
2
+ kernels : 186566
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 26624 ultraep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/run.env ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=ultraep
2
+ config=G4_K6
3
+ family=tierA
4
+ source_model=
5
+ standard_num_layers=
6
+ base_rung=G4
7
+ repo=/workspace/Megatron-UltraEP
8
+ branch=ultraep-bracing-regions-through-kernels
9
+ commit=03d57ef7
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=896
14
+ num_experts=128
15
+ topk=6
16
+ experts_per_rank=16
17
+ expert_params=11010048
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ q_lora_rank=
27
+ kv_lora_rank=512
28
+ qk_head_dim=128
29
+ qk_pos_emb_head_dim=64
30
+ v_head_dim=128
31
+ dense_first_layers=0
32
+ moe_layers=16
33
+ rotary_base=1000000
34
+ rotary_percent=1.0
35
+ rotary_scaling=
36
+ mscale=
37
+ mscale_all_dim=
38
+ qkv_bias=0
39
+ router_balancing=aux_loss
40
+ aux_loss_coeff=1e-2
41
+ router_pre_softmax=0
42
+ router_fusion=0
43
+ router_dtype=fp32
44
+ router_groups=
45
+ router_group_topk=
46
+ router_score_function=softmax
47
+ router_expert_bias=0
48
+ router_bias_update_rate=
49
+ router_topk_scaling_factor=
50
+ shared_expert_gate=0
51
+ apply_layernorm_1p=0
52
+ attention_output_gate=0
53
+ linear_attention_type=
54
+ linear_attention_freq=
55
+ linear_conv_kernel_dim=
56
+ linear_key_head_dim=
57
+ linear_value_head_dim=
58
+ linear_num_key_heads=
59
+ linear_num_value_heads=
60
+ mtp_num_layers=0
61
+ mtp_loss_scaling_factor=0.1
62
+ no_weight_decay_cond_type=
63
+ max_position_embeddings=32768
64
+ init_method_std=0.01
65
+ device_initiated_grouped_gemm=1
66
+ norm_epsilon=1e-5
67
+ seq_length=4096
68
+ micro_batch_size=1
69
+ global_batch_size=16
70
+ num_microbatches=2
71
+ assignments_per_rank=24576
72
+ mean_tokens_per_expert=1536
73
+ slot_policy=matched
74
+ slots_per_rank=16
75
+ capacity_mode=static
76
+ capacity=1.1354166666666667
77
+ cuda_graph=1
78
+ probe_mode=0
79
+ async_grad_reduce=0
80
+ branch_role=timing
81
+ memory_dump_mode=none
82
+ memory_snapshot_path=
83
+ cuda_graph_warmup_steps=3
84
+ recompute=none
85
+ force_balance=0
86
+ train_iters=10
87
+ ep_size=8
88
+ echo_dispatch_overlap=1
89
+ echo_dispatch_recompute=0
90
+ ultraep_quota_min_tokens=192
91
+ ultraep_weight_sync_join=
92
+ profile_enable=1
93
+ kernel_trace=1
94
+ fp8_mode=mxfp8
95
+ fp8_recipe=mxfp8
96
+ training_seed=1234
97
+ learning_rate=1e-4
98
+ min_learning_rate=1.0e-5
99
+ lr_warmup_iters=5
100
+ state_dump_enable=0
101
+ state_dump_dir=
102
+ moonep_assert_no_pending_reduce=0
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K6__ultraep__matched__seq4096__gbs16__cap1p1354166666666667/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/trace_results.tsv CHANGED
@@ -1,3 +1,6 @@
1
  model backend capacity status report async_overlap_json
2
  G2_K2 ultraep 1.15625 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625/nsys/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625.nsys-rep
3
  G3_K2 ultraep 1.1875 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875/nsys/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875.nsys-rep
 
 
 
 
1
  model backend capacity status report async_overlap_json
2
  G2_K2 ultraep 1.15625 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625/nsys/G2_K2__ultraep__matched__seq4096__gbs16__cap1p15625.nsys-rep
3
  G3_K2 ultraep 1.1875 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875/nsys/G3_K2__ultraep__matched__seq4096__gbs16__cap1p1875.nsys-rep
4
+ G3_K8 ultraep 1.09375 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375/nsys/G3_K8__ultraep__matched__seq4096__gbs16__cap1p09375.nsys-rep
5
+ G4_K2 ultraep 1.265625 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625/nsys/G4_K2__ultraep__matched__seq4096__gbs16__cap1p265625.nsys-rep
6
+ G4_K4 ultraep 1.171875 OK /workspace/granularity_next_b200/tiera_topk_trace/ultraep/tierbtrace/tierbtrace_gbs16/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875/nsys/G4_K4__ultraep__matched__seq4096__gbs16__cap1p171875.nsys-rep