AbhishekGhosh1998 commited on
Commit
3f0e96d
·
verified ·
1 Parent(s): f3ba99a

granularity results tiera-rerun-fixed-moonep @ 2026-08-30T09:43:55Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 (part 2)

Browse files
Files changed (25) hide show
  1. .gitattributes +3 -0
  2. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
  3. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
  4. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
  5. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
  6. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +65 -0
  7. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
  8. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
  9. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
  10. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
  11. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
  12. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +73 -0
  13. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
  14. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
  15. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
  16. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
  17. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
  18. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
  19. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +73 -0
  20. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
  21. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
  22. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +73 -0
  23. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +1382 -0
  24. tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv +3 -0
  25. tiera-rerun-fixed-moonep/study.log +16 -0
.gitattributes CHANGED
@@ -78,3 +78,6 @@ tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G1__ultraep__matche
78
  tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/G2__ultraep__matched__seq4096__gbs16__cap1p08203125.nsys-rep filter=lfs diff=lfs merge=lfs -text
79
  tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625/nsys/G3__ultraep__matched__seq4096__gbs16__cap1p072265625.nsys-rep filter=lfs diff=lfs merge=lfs -text
80
  tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/nsys/G4__ultraep__matched__seq4096__gbs16__cap1p095703125.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
 
 
78
  tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/G2__ultraep__matched__seq4096__gbs16__cap1p08203125.nsys-rep filter=lfs diff=lfs merge=lfs -text
79
  tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625/nsys/G3__ultraep__matched__seq4096__gbs16__cap1p072265625.nsys-rep filter=lfs diff=lfs merge=lfs -text
80
  tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/nsys/G4__ultraep__matched__seq4096__gbs16__cap1p095703125.nsys-rep filter=lfs diff=lfs merge=lfs -text
81
+ tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
82
+ tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
83
+ tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d454bc930b451579b46b31a3e7de18980bfce3e0e66ed011dab8a879f5052faa
3
+ size 9671848
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 109.24575,
4
+ "overlapped_ms_per_iteration": 58.440625,
5
+ "exposed_ms_per_iteration": 50.805125,
6
+ "overlap_pct": 53.49464395640105,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 232.537 ms device_kernel
4
+ REDUCE 155.815 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
5
+ 109.559 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
6
+ REDUCE 62.716 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
7
+ 52.685 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
8
+ 50.905 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
9
+ 47.933 ms multi_tensor_apply_kernel
10
+ 36.710 ms cast_mxfp8_2D_kernel
11
+ 32.449 ms elementwise_kernel
12
+ 30.104 ms vectorized_elementwise_kernel
13
+ 28.983 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
14
+ 27.574 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
15
+ 25.304 ms triton_poi_fused__to_copy_cat_0
16
+ 21.589 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
17
+ 16.731 ms unrolled_elementwise_kernel
18
+ 16.082 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
19
+ 10.482 ms ncclDevKernel_AllGather_RING_LL
20
+ 10.454 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
21
+ 8.708 ms setGroupedGemmWgradArguments
22
+ 8.183 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
23
+ 7.737 ms triton_red_fused__to_copy_mul_silu_sum_1
24
+ 7.632 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
25
+ 7.015 ms triton_poi_fused__to_copy_mul_silu_0
26
+ 5.923 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
27
+ 5.783 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
28
+ 4.431 ms rmsnorm_bwd_tuned_kernel
29
+ 3.961 ms CatArrayBatchedCopy_alignedK_contig
30
+ 3.938 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
31
+ 3.821 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
32
+ 3.595 ms reduce_kernel
33
+ 3.508 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
34
+ 3.414 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
35
+ 3.111 ms fused_rope_backward_kernel
36
+ 2.845 ms _scatter_gather_elementwise_kernel
37
+ 2.606 ms multi_tensor_swizzle_row_scaling_kernel
38
+ 2.447 ms fused_rope_forward_kernel
39
+ 2.420 ms multi_tensor_swizzle_col_scaling_kernel
40
+ 2.267 ms swizzle_col_scaling_kernel
41
+ 1.886 ms rmsnorm_fwd_tuned_kernel
42
+ 1.790 ms triton_poi_fused_add_0
43
+
44
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 128 (64/iter)
46
+ reduce_grad sum : 109.265 ms/iter
47
+ reduce_grad busy (union) : 109.265 ms/iter
48
+ ... overlapped with compute: 58.042 ms/iter ( 53.1 %)
49
+ ... exposed on critical path: 51.223 ms/iter ( 46.9 %)
50
+ other GPU work (union) : 425.134 ms/iter
51
+ traced wall : 494.975 ms/iter
52
+ --> time hidden by overlap : 58.042 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 232.734 ms device_kernel
56
+ REDUCE 154.081 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
57
+ 92.807 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
58
+ 75.101 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
59
+ REDUCE 63.763 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
60
+ 48.211 ms multi_tensor_apply_kernel
61
+ 46.010 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
62
+ 36.926 ms cast_mxfp8_2D_kernel
63
+ 33.138 ms elementwise_kernel
64
+ 30.031 ms vectorized_elementwise_kernel
65
+ 29.104 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
66
+ 27.997 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
67
+ 25.084 ms triton_poi_fused__to_copy_cat_0
68
+ 23.511 ms ncclDevKernel_AllGather_RING_LL
69
+ 21.537 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
70
+ 17.051 ms unrolled_elementwise_kernel
71
+ 15.337 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
72
+ 10.482 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
73
+ 8.678 ms setGroupedGemmWgradArguments
74
+ 8.186 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
75
+ 7.741 ms triton_red_fused__to_copy_mul_silu_sum_1
76
+ 7.614 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
77
+ 7.079 ms triton_poi_fused__to_copy_mul_silu_0
78
+ 5.944 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
79
+ 5.793 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
80
+ 4.598 ms rmsnorm_bwd_tuned_kernel
81
+ 3.971 ms CatArrayBatchedCopy_alignedK_contig
82
+ 3.921 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
83
+ 3.781 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
84
+ 3.596 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
85
+ 3.514 ms reduce_kernel
86
+ 3.416 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
87
+ 3.143 ms fused_rope_backward_kernel
88
+ 2.816 ms _scatter_gather_elementwise_kernel
89
+ 2.594 ms multi_tensor_swizzle_row_scaling_kernel
90
+ 2.566 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
91
+ 2.457 ms fused_rope_forward_kernel
92
+ 2.450 ms multi_tensor_swizzle_col_scaling_kernel
93
+ 2.243 ms swizzle_col_scaling_kernel
94
+ 1.893 ms rmsnorm_fwd_tuned_kernel
95
+
96
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 128 (64/iter)
98
+ reduce_grad sum : 108.922 ms/iter
99
+ reduce_grad busy (union) : 108.922 ms/iter
100
+ ... overlapped with compute: 58.675 ms/iter ( 53.9 %)
101
+ ... exposed on critical path: 50.247 ms/iter ( 46.1 %)
102
+ other GPU work (union) : 433.322 ms/iter
103
+ traced wall : 501.495 ms/iter
104
+ --> time hidden by overlap : 58.675 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 230.190 ms device_kernel
108
+ REDUCE 154.378 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
109
+ 94.558 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
110
+ 74.068 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
111
+ REDUCE 63.532 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
112
+ 48.174 ms multi_tensor_apply_kernel
113
+ 47.151 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
114
+ 37.000 ms cast_mxfp8_2D_kernel
115
+ 33.122 ms elementwise_kernel
116
+ 30.169 ms vectorized_elementwise_kernel
117
+ 29.298 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
118
+ 27.164 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
119
+ 24.908 ms triton_poi_fused__to_copy_cat_0
120
+ 21.530 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
121
+ 17.184 ms unrolled_elementwise_kernel
122
+ 15.745 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
123
+ 10.554 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
124
+ 8.606 ms setGroupedGemmWgradArguments
125
+ 8.362 ms ncclDevKernel_AllGather_RING_LL
126
+ 8.218 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
127
+ 7.740 ms triton_red_fused__to_copy_mul_silu_sum_1
128
+ 7.696 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
129
+ 7.022 ms triton_poi_fused__to_copy_mul_silu_0
130
+ 5.989 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
131
+ 5.838 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
132
+ 4.547 ms rmsnorm_bwd_tuned_kernel
133
+ 3.985 ms CatArrayBatchedCopy_alignedK_contig
134
+ 3.848 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
135
+ 3.830 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
136
+ 3.654 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
137
+ 3.535 ms reduce_kernel
138
+ 3.416 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
139
+ 3.134 ms fused_rope_backward_kernel
140
+ 2.862 ms _scatter_gather_elementwise_kernel
141
+ 2.622 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
142
+ 2.588 ms multi_tensor_swizzle_row_scaling_kernel
143
+ 2.476 ms fused_rope_forward_kernel
144
+ 2.406 ms multi_tensor_swizzle_col_scaling_kernel
145
+ 2.264 ms swizzle_col_scaling_kernel
146
+ 1.892 ms rmsnorm_fwd_tuned_kernel
147
+
148
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 128 (64/iter)
150
+ reduce_grad sum : 108.955 ms/iter
151
+ reduce_grad busy (union) : 108.955 ms/iter
152
+ ... overlapped with compute: 58.603 ms/iter ( 53.8 %)
153
+ ... exposed on critical path: 50.352 ms/iter ( 46.2 %)
154
+ other GPU work (union) : 425.739 ms/iter
155
+ traced wall : 494.216 ms/iter
156
+ --> time hidden by overlap : 58.603 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 234.150 ms device_kernel
160
+ REDUCE 156.439 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
161
+ 111.851 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
162
+ REDUCE 62.585 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
163
+ 55.020 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
164
+ 48.161 ms multi_tensor_apply_kernel
165
+ 43.500 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
166
+ 36.850 ms cast_mxfp8_2D_kernel
167
+ 33.223 ms elementwise_kernel
168
+ 30.624 ms vectorized_elementwise_kernel
169
+ 29.179 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
170
+ 27.485 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
171
+ 25.197 ms triton_poi_fused__to_copy_cat_0
172
+ 21.574 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
173
+ 17.082 ms unrolled_elementwise_kernel
174
+ 15.473 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
175
+ 10.525 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
176
+ 8.869 ms ncclDevKernel_AllGather_RING_LL
177
+ 8.734 ms setGroupedGemmWgradArguments
178
+ 8.173 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
179
+ 7.755 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
180
+ 7.738 ms triton_red_fused__to_copy_mul_silu_sum_1
181
+ 7.067 ms triton_poi_fused__to_copy_mul_silu_0
182
+ 5.965 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
183
+ 5.821 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
184
+ 4.563 ms rmsnorm_bwd_tuned_kernel
185
+ 3.978 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
186
+ 3.968 ms CatArrayBatchedCopy_alignedK_contig
187
+ 3.837 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
188
+ 3.638 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
189
+ 3.514 ms reduce_kernel
190
+ 3.450 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
191
+ 3.130 ms fused_rope_backward_kernel
192
+ 2.838 ms _scatter_gather_elementwise_kernel
193
+ 2.686 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
194
+ 2.638 ms multi_tensor_swizzle_row_scaling_kernel
195
+ 2.482 ms fused_rope_forward_kernel
196
+ 2.454 ms multi_tensor_swizzle_col_scaling_kernel
197
+ 2.262 ms swizzle_col_scaling_kernel
198
+ 1.893 ms rmsnorm_fwd_tuned_kernel
199
+
200
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 128 (64/iter)
202
+ reduce_grad sum : 109.512 ms/iter
203
+ reduce_grad busy (union) : 109.512 ms/iter
204
+ ... overlapped with compute: 58.530 ms/iter ( 53.4 %)
205
+ ... exposed on critical path: 50.982 ms/iter ( 46.6 %)
206
+ other GPU work (union) : 425.696 ms/iter
207
+ traced wall : 531.982 ms/iter
208
+ --> time hidden by overlap : 58.530 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 232.303 ms device_kernel
212
+ REDUCE 156.283 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
213
+ 109.634 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
214
+ REDUCE 62.579 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
215
+ 53.627 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
216
+ 49.043 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
217
+ 48.214 ms multi_tensor_apply_kernel
218
+ 36.810 ms cast_mxfp8_2D_kernel
219
+ 33.300 ms elementwise_kernel
220
+ 30.213 ms vectorized_elementwise_kernel
221
+ 29.279 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
222
+ 27.478 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
223
+ 25.244 ms triton_poi_fused__to_copy_cat_0
224
+ 21.467 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
225
+ 17.089 ms unrolled_elementwise_kernel
226
+ 15.643 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
227
+ 10.591 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
228
+ 8.966 ms ncclDevKernel_AllGather_RING_LL
229
+ 8.875 ms setGroupedGemmWgradArguments
230
+ 8.197 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
231
+ 7.742 ms triton_red_fused__to_copy_mul_silu_sum_1
232
+ 7.706 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
233
+ 7.067 ms triton_poi_fused__to_copy_mul_silu_0
234
+ 5.993 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
235
+ 5.843 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
236
+ 4.560 ms rmsnorm_bwd_tuned_kernel
237
+ 4.020 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
238
+ 3.974 ms CatArrayBatchedCopy_alignedK_contig
239
+ 3.832 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
240
+ 3.618 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
241
+ 3.548 ms reduce_kernel
242
+ 3.456 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
243
+ 3.128 ms fused_rope_backward_kernel
244
+ 2.855 ms _scatter_gather_elementwise_kernel
245
+ 2.626 ms multi_tensor_swizzle_row_scaling_kernel
246
+ 2.457 ms fused_rope_forward_kernel
247
+ 2.450 ms multi_tensor_swizzle_col_scaling_kernel
248
+ 2.262 ms swizzle_col_scaling_kernel
249
+ 1.893 ms rmsnorm_fwd_tuned_kernel
250
+ 1.815 ms triton_poi_fused_add_0
251
+
252
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 128 (64/iter)
254
+ reduce_grad sum : 109.431 ms/iter
255
+ reduce_grad busy (union) : 109.431 ms/iter
256
+ ... overlapped with compute: 58.611 ms/iter ( 53.6 %)
257
+ ... exposed on critical path: 50.820 ms/iter ( 46.4 %)
258
+ other GPU work (union) : 424.447 ms/iter
259
+ traced wall : 494.369 ms/iter
260
+ --> time hidden by overlap : 58.611 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 229.723 ms device_kernel
264
+ REDUCE 153.790 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
265
+ 87.114 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
266
+ 85.452 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
267
+ REDUCE 63.862 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
268
+ 48.234 ms multi_tensor_apply_kernel
269
+ 44.522 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
270
+ 37.040 ms cast_mxfp8_2D_kernel
271
+ 33.068 ms elementwise_kernel
272
+ 29.991 ms vectorized_elementwise_kernel
273
+ 29.293 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
274
+ 27.474 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
275
+ 24.920 ms triton_poi_fused__to_copy_cat_0
276
+ 21.505 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
277
+ 17.069 ms unrolled_elementwise_kernel
278
+ 15.453 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
279
+ 10.841 ms ncclDevKernel_AllGather_RING_LL
280
+ 10.531 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
281
+ 8.765 ms setGroupedGemmWgradArguments
282
+ 8.263 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
283
+ 7.742 ms triton_red_fused__to_copy_mul_silu_sum_1
284
+ 7.719 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
285
+ 7.046 ms triton_poi_fused__to_copy_mul_silu_0
286
+ 5.984 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
287
+ 5.834 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
288
+ 4.605 ms rmsnorm_bwd_tuned_kernel
289
+ 3.979 ms CatArrayBatchedCopy_alignedK_contig
290
+ 3.875 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
291
+ 3.816 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
292
+ 3.596 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
293
+ 3.511 ms reduce_kernel
294
+ 3.376 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
295
+ 3.144 ms fused_rope_backward_kernel
296
+ 2.925 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
297
+ 2.823 ms _scatter_gather_elementwise_kernel
298
+ 2.582 ms multi_tensor_swizzle_row_scaling_kernel
299
+ 2.475 ms fused_rope_forward_kernel
300
+ 2.424 ms multi_tensor_swizzle_col_scaling_kernel
301
+ 2.242 ms swizzle_col_scaling_kernel
302
+ 1.899 ms rmsnorm_fwd_tuned_kernel
303
+
304
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 128 (64/iter)
306
+ reduce_grad sum : 108.826 ms/iter
307
+ reduce_grad busy (union) : 108.826 ms/iter
308
+ ... overlapped with compute: 58.653 ms/iter ( 53.9 %)
309
+ ... exposed on critical path: 50.173 ms/iter ( 46.1 %)
310
+ other GPU work (union) : 427.265 ms/iter
311
+ traced wall : 495.128 ms/iter
312
+ --> time hidden by overlap : 58.653 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 231.663 ms device_kernel
316
+ REDUCE 157.279 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
317
+ 113.340 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
318
+ REDUCE 62.173 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
319
+ 55.110 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
320
+ 48.219 ms multi_tensor_apply_kernel
321
+ 44.531 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
322
+ 36.828 ms cast_mxfp8_2D_kernel
323
+ 33.165 ms elementwise_kernel
324
+ 29.799 ms vectorized_elementwise_kernel
325
+ 29.089 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
326
+ 27.346 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
327
+ 25.188 ms triton_poi_fused__to_copy_cat_0
328
+ 21.527 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
329
+ 16.942 ms unrolled_elementwise_kernel
330
+ 15.869 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
331
+ 10.454 ms ncclDevKernel_AllGather_RING_LL
332
+ 10.268 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
333
+ 8.796 ms setGroupedGemmWgradArguments
334
+ 8.181 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
335
+ 7.739 ms triton_red_fused__to_copy_mul_silu_sum_1
336
+ 7.603 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
337
+ 7.013 ms triton_poi_fused__to_copy_mul_silu_0
338
+ 5.913 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
339
+ 5.780 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
340
+ 4.526 ms rmsnorm_bwd_tuned_kernel
341
+ 3.944 ms CatArrayBatchedCopy_alignedK_contig
342
+ 3.881 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
343
+ 3.832 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
344
+ 3.598 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
345
+ 3.401 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
346
+ 3.374 ms reduce_kernel
347
+ 3.114 ms fused_rope_backward_kernel
348
+ 2.784 ms _scatter_gather_elementwise_kernel
349
+ 2.714 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
350
+ 2.622 ms multi_tensor_swizzle_row_scaling_kernel
351
+ 2.453 ms fused_rope_forward_kernel
352
+ 2.426 ms multi_tensor_swizzle_col_scaling_kernel
353
+ 2.236 ms swizzle_col_scaling_kernel
354
+ 1.883 ms rmsnorm_fwd_tuned_kernel
355
+
356
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 128 (64/iter)
358
+ reduce_grad sum : 109.726 ms/iter
359
+ reduce_grad busy (union) : 109.726 ms/iter
360
+ ... overlapped with compute: 57.996 ms/iter ( 52.9 %)
361
+ ... exposed on critical path: 51.730 ms/iter ( 47.1 %)
362
+ other GPU work (union) : 425.576 ms/iter
363
+ traced wall : 494.923 ms/iter
364
+ --> time hidden by overlap : 57.996 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 229.957 ms device_kernel
368
+ REDUCE 155.726 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
369
+ 105.501 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
370
+ REDUCE 62.933 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
371
+ 58.493 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
372
+ 52.107 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0
373
+ 48.226 ms multi_tensor_apply_kernel
374
+ 36.808 ms cast_mxfp8_2D_kernel
375
+ 33.145 ms elementwise_kernel
376
+ 30.131 ms vectorized_elementwise_kernel
377
+ 29.044 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
378
+ 27.840 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
379
+ 24.994 ms triton_poi_fused__to_copy_cat_0
380
+ 21.499 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0
381
+ 17.114 ms unrolled_elementwise_kernel
382
+ 15.468 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0
383
+ 10.369 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
384
+ 8.685 ms setGroupedGemmWgradArguments
385
+ 8.319 ms ncclDevKernel_AllGather_RING_LL
386
+ 8.162 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
387
+ 7.740 ms triton_red_fused__to_copy_mul_silu_sum_1
388
+ 7.678 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
389
+ 6.989 ms triton_poi_fused__to_copy_mul_silu_0
390
+ 5.918 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
391
+ 5.771 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
392
+ 4.555 ms rmsnorm_bwd_tuned_kernel
393
+ 3.980 ms CatArrayBatchedCopy_alignedK_contig
394
+ 3.875 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
395
+ 3.841 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
396
+ 3.643 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0
397
+ 3.561 ms reduce_kernel
398
+ 3.374 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
399
+ 3.132 ms fused_rope_backward_kernel
400
+ 2.874 ms _scatter_gather_elementwise_kernel
401
+ 2.596 ms multi_tensor_swizzle_row_scaling_kernel
402
+ 2.461 ms fused_rope_forward_kernel
403
+ 2.414 ms multi_tensor_swizzle_col_scaling_kernel
404
+ 2.413 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
405
+ 2.273 ms swizzle_col_scaling_kernel
406
+ 1.897 ms rmsnorm_fwd_tuned_kernel
407
+
408
+ === G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 128 (64/iter)
410
+ reduce_grad sum : 109.329 ms/iter
411
+ reduce_grad busy (union) : 109.329 ms/iter
412
+ ... overlapped with compute: 58.415 ms/iter ( 53.4 %)
413
+ ... exposed on critical path: 50.914 ms/iter ( 46.6 %)
414
+ other GPU work (union) : 425.110 ms/iter
415
+ traced wall : 494.224 ms/iter
416
+ --> time hidden by overlap : 58.415 ms/iter
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=109.246 ms/iter overlap=58.441 ms/iter (53.5%) exposed=50.805 ms/iter status=PASS
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log CHANGED
@@ -865,3 +865,68 @@ CUDA graph capture done!!!
865
  [2026-08-30 09:33:41] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 500.8 | throughput per GPU (TFLOP/s/GPU): 665.7 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.012681E+01 | loss scale: 1.0 | grad norm: 43.423 | number of skipped iterations: 0 | number of nan iterations: 0 |
866
  [2026-08-30 09:33:44] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3206.3 | throughput per GPU (TFLOP/s/GPU): 104.0 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.871505E+00 | loss scale: 1.0 | grad norm: 7.989 | number of skipped iterations: 0 | number of nan iterations: 0 |
867
  [2026-08-30 09:33:44] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 487.6 | throughput per GPU (TFLOP/s/GPU): 683.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.080805E+00 | loss scale: 1.0 | grad norm: 6.615 | number of skipped iterations: 0 | number of nan iterations: 0 |
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
865
  [2026-08-30 09:33:41] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 500.8 | throughput per GPU (TFLOP/s/GPU): 665.7 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.012681E+01 | loss scale: 1.0 | grad norm: 43.423 | number of skipped iterations: 0 | number of nan iterations: 0 |
866
  [2026-08-30 09:33:44] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3206.3 | throughput per GPU (TFLOP/s/GPU): 104.0 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.871505E+00 | loss scale: 1.0 | grad norm: 7.989 | number of skipped iterations: 0 | number of nan iterations: 0 |
867
  [2026-08-30 09:33:44] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 487.6 | throughput per GPU (TFLOP/s/GPU): 683.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.080805E+00 | loss scale: 1.0 | grad norm: 6.615 | number of skipped iterations: 0 | number of nan iterations: 0 |
868
+ Capture range started in the application.
869
+ Capture range ended in the application.
870
+ Generating '/tmp/nsys-report-c110.qdstrm'
871
+
872
+
873
+ [2026-08-30 09:34:27] iteration 9/ 10 | consumed samples: 144 | elapsed time per iteration (ms): 42485.7 | throughput per GPU (TFLOP/s/GPU): 7.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.706870E+00 | loss scale: 1.0 | grad norm: 6.800 | number of skipped iterations: 0 | number of nan iterations: 0 |
874
+ [2026-08-30 09:34:27] iteration 10/ 10 | consumed samples: 160 | elapsed time per iteration (ms): 508.0 | throughput per GPU (TFLOP/s/GPU): 656.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.508333E+00 | loss scale: 1.0 | grad norm: 5.796 | number of skipped iterations: 0 | number of nan iterations: 0 |
875
+ [after training is done] datetime: 2026-08-30 09:34:27
876
+ [rank0]:[W830 09:34:29.253133092 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
877
+ [rank6]:[W830 09:34:30.434756072 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
878
+ [rank4]:[W830 09:34:30.439653428 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
879
+ [rank3]:[W830 09:34:30.501548768 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
880
+ [rank1]:[W830 09:34:30.710457488 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
881
+ [rank5]:[W830 09:34:30.894298920 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
882
+ [rank7]:[W830 09:34:30.062143723 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
883
+ [rank2]:[W830 09:34:31.620052340 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
884
+ [rank3]:[W830 09:34:33.744760822 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
885
+ pure virtual method called
886
+ terminate called without an active exception
887
+ [rank0]:[W830 09:34:33.901720434 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
888
+ pure virtual method called
889
+ terminate called without an active exception
890
+ Collection stop is not allowed in this state.
891
+ W0830 09:34:34.074000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933075 closing signal SIGTERM
892
+ W0830 09:34:34.074000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933076 closing signal SIGTERM
893
+ W0830 09:34:34.075000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933077 closing signal SIGTERM
894
+ W0830 09:34:34.076000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933079 closing signal SIGTERM
895
+ W0830 09:34:34.078000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933080 closing signal SIGTERM
896
+ W0830 09:34:34.079000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933081 closing signal SIGTERM
897
+ W0830 09:34:34.080000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933082 closing signal SIGTERM
898
+ E0830 09:34:35.739000 932866 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 3 (pid: 933078) of binary: /usr/bin/python
899
+ Traceback (most recent call last):
900
+ File "/usr/local/bin/torchrun", line 8, in <module>
901
+ sys.exit(main())
902
+ ^^^^^^
903
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
904
+ return f(*args, **kwargs)
905
+ ^^^^^^^^^^^^^^^^^^
906
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
907
+ run(args)
908
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
909
+ elastic_launch(
910
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
911
+ return launch_agent(self._config, self._entrypoint, list(args))
912
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
913
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
914
+ raise ChildFailedError(
915
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
916
+ ============================================================
917
+ pretrain_gpt.py FAILED
918
+ ------------------------------------------------------------
919
+ Failures:
920
+ <NO_OTHER_FAILURES>
921
+ ------------------------------------------------------------
922
+ Root Cause (first observed failure):
923
+ [0]:
924
+ time : 2026-08-30_09:34:34
925
+ host : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
926
+ rank : 3 (local_rank: 3)
927
+ exitcode : -6 (pid: 933078)
928
+ error_file: <N/A>
929
+ traceback : Signal 6 (SIGABRT) received by PID 933078
930
+ ============================================================
931
+ Generated:
932
+ /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
2
+ kernels : 127130
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4dd9f456bc11bcad68e1f2d55a124e50fcad6be13301cff2443f566104fa392d
3
+ size 10443380
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 28.751875,
4
+ "overlapped_ms_per_iteration": 27.93925,
5
+ "exposed_ms_per_iteration": 0.812625,
6
+ "overlap_pct": 97.17366258722258,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 222.064 ms device_kernel
4
+ 94.247 ms elementwise_kernel
5
+ 58.260 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
6
+ 48.132 ms multi_tensor_apply_kernel
7
+ 48.102 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
8
+ 46.940 ms cast_mxfp8_2D_kernel
9
+ 37.835 ms vectorized_elementwise_kernel
10
+ REDUCE 37.816 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
11
+ 29.767 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
12
+ 27.990 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
13
+ 25.630 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
14
+ 24.873 ms triton_poi_fused__to_copy_cat_0
15
+ REDUCE 20.247 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
16
+ 19.705 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
17
+ 16.659 ms unrolled_elementwise_kernel
18
+ 12.134 ms ncclDevKernel_AllGather_RING_LL
19
+ 9.957 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
20
+ 9.474 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
21
+ 8.859 ms triton_red_fused__to_copy_mul_silu_sum_1
22
+ 8.218 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
23
+ 7.734 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
24
+ 6.919 ms triton_poi_fused__to_copy_mul_silu_0
25
+ 5.442 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
26
+ 5.301 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
27
+ 4.847 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
28
+ 4.340 ms setGroupedGemmWgradArguments
29
+ 3.950 ms rmsnorm_bwd_tuned_kernel
30
+ 3.939 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
31
+ 3.838 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
32
+ 3.814 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
33
+ 3.564 ms CatArrayBatchedCopy_alignedK_contig
34
+ 3.496 ms reduce_kernel
35
+ 3.397 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
36
+ 3.390 ms _scatter_gather_elementwise_kernel
37
+ 3.212 ms multi_tensor_swizzle_row_scaling_kernel
38
+ 2.925 ms fused_rope_backward_kernel
39
+ 2.766 ms multi_tensor_swizzle_col_scaling_kernel
40
+ 2.458 ms fused_rope_forward_kernel
41
+ 2.296 ms swizzle_col_scaling_kernel
42
+ 1.944 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0
43
+
44
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 128 (64/iter)
46
+ reduce_grad sum : 29.032 ms/iter
47
+ reduce_grad busy (union) : 29.032 ms/iter
48
+ ... overlapped with compute: 28.236 ms/iter ( 97.3 %)
49
+ ... exposed on critical path: 0.795 ms/iter ( 2.7 %)
50
+ other GPU work (union) : 406.726 ms/iter
51
+ traced wall : 444.786 ms/iter
52
+ --> time hidden by overlap : 28.236 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 218.011 ms device_kernel
56
+ 96.436 ms elementwise_kernel
57
+ 58.112 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
58
+ 48.773 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
59
+ 48.187 ms multi_tensor_apply_kernel
60
+ 46.749 ms cast_mxfp8_2D_kernel
61
+ 37.711 ms vectorized_elementwise_kernel
62
+ REDUCE 37.280 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
63
+ 29.240 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
64
+ 27.544 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
65
+ 25.457 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
66
+ 24.726 ms triton_poi_fused__to_copy_cat_0
67
+ 21.229 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
68
+ REDUCE 19.806 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
69
+ 16.659 ms unrolled_elementwise_kernel
70
+ 10.405 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
71
+ 9.498 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
72
+ 9.148 ms ncclDevKernel_AllGather_RING_LL
73
+ 8.830 ms triton_red_fused__to_copy_mul_silu_sum_1
74
+ 8.119 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
75
+ 7.641 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
76
+ 6.980 ms triton_poi_fused__to_copy_mul_silu_0
77
+ 5.968 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
78
+ 5.418 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
79
+ 5.227 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
80
+ 4.460 ms setGroupedGemmWgradArguments
81
+ 3.970 ms rmsnorm_bwd_tuned_kernel
82
+ 3.921 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
83
+ 3.905 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
84
+ 3.800 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
85
+ 3.557 ms CatArrayBatchedCopy_alignedK_contig
86
+ 3.482 ms reduce_kernel
87
+ 3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
88
+ 3.319 ms _scatter_gather_elementwise_kernel
89
+ 3.284 ms multi_tensor_swizzle_row_scaling_kernel
90
+ 2.925 ms fused_rope_backward_kernel
91
+ 2.786 ms multi_tensor_swizzle_col_scaling_kernel
92
+ 2.451 ms fused_rope_forward_kernel
93
+ 2.253 ms swizzle_col_scaling_kernel
94
+ 1.988 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0
95
+
96
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 128 (64/iter)
98
+ reduce_grad sum : 28.543 ms/iter
99
+ reduce_grad busy (union) : 28.543 ms/iter
100
+ ... overlapped with compute: 27.703 ms/iter ( 97.1 %)
101
+ ... exposed on critical path: 0.840 ms/iter ( 2.9 %)
102
+ other GPU work (union) : 405.442 ms/iter
103
+ traced wall : 443.732 ms/iter
104
+ --> time hidden by overlap : 27.703 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 216.307 ms device_kernel
108
+ 96.416 ms elementwise_kernel
109
+ 58.084 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
110
+ 48.194 ms multi_tensor_apply_kernel
111
+ 48.124 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
112
+ 46.869 ms cast_mxfp8_2D_kernel
113
+ 37.953 ms vectorized_elementwise_kernel
114
+ REDUCE 36.718 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
115
+ 29.144 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
116
+ 27.821 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
117
+ 26.502 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
118
+ 24.724 ms triton_poi_fused__to_copy_cat_0
119
+ 21.168 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
120
+ REDUCE 20.391 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
121
+ 16.773 ms unrolled_elementwise_kernel
122
+ 10.313 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
123
+ 10.204 ms ncclDevKernel_AllGather_RING_LL
124
+ 9.486 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
125
+ 8.838 ms triton_red_fused__to_copy_mul_silu_sum_1
126
+ 8.212 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
127
+ 7.758 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
128
+ 6.947 ms triton_poi_fused__to_copy_mul_silu_0
129
+ 6.377 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
130
+ 5.442 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
131
+ 5.263 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
132
+ 4.315 ms setGroupedGemmWgradArguments
133
+ 3.960 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
134
+ 3.945 ms rmsnorm_bwd_tuned_kernel
135
+ 3.848 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
136
+ 3.837 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
137
+ 3.552 ms CatArrayBatchedCopy_alignedK_contig
138
+ 3.519 ms reduce_kernel
139
+ 3.353 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
140
+ 3.344 ms _scatter_gather_elementwise_kernel
141
+ 3.225 ms multi_tensor_swizzle_row_scaling_kernel
142
+ 2.835 ms fused_rope_backward_kernel
143
+ 2.743 ms multi_tensor_swizzle_col_scaling_kernel
144
+ 2.455 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0
145
+ 2.452 ms fused_rope_forward_kernel
146
+ 2.279 ms swizzle_col_scaling_kernel
147
+
148
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 128 (64/iter)
150
+ reduce_grad sum : 28.554 ms/iter
151
+ reduce_grad busy (union) : 28.554 ms/iter
152
+ ... overlapped with compute: 27.734 ms/iter ( 97.1 %)
153
+ ... exposed on critical path: 0.820 ms/iter ( 2.9 %)
154
+ other GPU work (union) : 406.207 ms/iter
155
+ traced wall : 443.971 ms/iter
156
+ --> time hidden by overlap : 27.734 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 219.931 ms device_kernel
160
+ 96.310 ms elementwise_kernel
161
+ 58.117 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
162
+ 48.923 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
163
+ 48.072 ms multi_tensor_apply_kernel
164
+ 46.855 ms cast_mxfp8_2D_kernel
165
+ 37.473 ms vectorized_elementwise_kernel
166
+ REDUCE 36.402 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
167
+ 29.275 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
168
+ 28.747 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
169
+ 27.786 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
170
+ 24.749 ms triton_poi_fused__to_copy_cat_0
171
+ REDUCE 20.723 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
172
+ 17.471 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
173
+ 16.621 ms unrolled_elementwise_kernel
174
+ 10.273 ms ncclDevKernel_AllGather_RING_LL
175
+ 9.475 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
176
+ 8.824 ms triton_red_fused__to_copy_mul_silu_sum_1
177
+ 8.582 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
178
+ 8.193 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
179
+ 7.821 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
180
+ 6.990 ms triton_poi_fused__to_copy_mul_silu_0
181
+ 6.932 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
182
+ 5.464 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
183
+ 5.214 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
184
+ 4.492 ms setGroupedGemmWgradArguments
185
+ 3.980 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
186
+ 3.963 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
187
+ 3.930 ms rmsnorm_bwd_tuned_kernel
188
+ 3.840 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
189
+ 3.535 ms CatArrayBatchedCopy_alignedK_contig
190
+ 3.486 ms reduce_kernel
191
+ 3.451 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
192
+ 3.308 ms _scatter_gather_elementwise_kernel
193
+ 3.203 ms multi_tensor_swizzle_row_scaling_kernel
194
+ 2.867 ms fused_rope_backward_kernel
195
+ 2.791 ms multi_tensor_swizzle_col_scaling_kernel
196
+ 2.459 ms fused_rope_forward_kernel
197
+ 2.265 ms swizzle_col_scaling_kernel
198
+ 1.906 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0
199
+
200
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 128 (64/iter)
202
+ reduce_grad sum : 28.562 ms/iter
203
+ reduce_grad busy (union) : 28.562 ms/iter
204
+ ... overlapped with compute: 27.750 ms/iter ( 97.2 %)
205
+ ... exposed on critical path: 0.812 ms/iter ( 2.8 %)
206
+ other GPU work (union) : 406.509 ms/iter
207
+ traced wall : 443.946 ms/iter
208
+ --> time hidden by overlap : 27.750 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 221.823 ms device_kernel
212
+ 96.605 ms elementwise_kernel
213
+ 58.051 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
214
+ 48.697 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
215
+ 48.199 ms multi_tensor_apply_kernel
216
+ 46.896 ms cast_mxfp8_2D_kernel
217
+ REDUCE 37.843 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
218
+ 37.585 ms vectorized_elementwise_kernel
219
+ 29.295 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
220
+ 27.894 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
221
+ 24.762 ms triton_poi_fused__to_copy_cat_0
222
+ 24.376 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
223
+ REDUCE 19.707 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
224
+ 16.800 ms unrolled_elementwise_kernel
225
+ 16.589 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
226
+ 11.394 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
227
+ 10.496 ms ncclDevKernel_AllGather_RING_LL
228
+ 9.709 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
229
+ 8.830 ms triton_red_fused__to_copy_mul_silu_sum_1
230
+ 8.232 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
231
+ 7.784 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
232
+ 7.022 ms triton_poi_fused__to_copy_mul_silu_0
233
+ 5.525 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
234
+ 5.282 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
235
+ 4.421 ms setGroupedGemmWgradArguments
236
+ 3.990 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
237
+ 3.964 ms rmsnorm_bwd_tuned_kernel
238
+ 3.940 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
239
+ 3.862 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
240
+ 3.536 ms CatArrayBatchedCopy_alignedK_contig
241
+ 3.476 ms reduce_kernel
242
+ 3.443 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
243
+ 3.309 ms _scatter_gather_elementwise_kernel
244
+ 3.294 ms multi_tensor_swizzle_row_scaling_kernel
245
+ 2.841 ms fused_rope_backward_kernel
246
+ 2.796 ms multi_tensor_swizzle_col_scaling_kernel
247
+ 2.462 ms fused_rope_forward_kernel
248
+ 2.273 ms swizzle_col_scaling_kernel
249
+ 1.876 ms rmsnorm_fwd_tuned_kernel
250
+ 1.777 ms gatherTopK
251
+
252
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 128 (64/iter)
254
+ reduce_grad sum : 28.775 ms/iter
255
+ reduce_grad busy (union) : 28.775 ms/iter
256
+ ... overlapped with compute: 27.984 ms/iter ( 97.3 %)
257
+ ... exposed on critical path: 0.791 ms/iter ( 2.7 %)
258
+ other GPU work (union) : 403.102 ms/iter
259
+ traced wall : 444.881 ms/iter
260
+ --> time hidden by overlap : 27.984 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 217.680 ms device_kernel
264
+ 96.334 ms elementwise_kernel
265
+ 58.085 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
266
+ 48.507 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
267
+ 48.151 ms multi_tensor_apply_kernel
268
+ 46.770 ms cast_mxfp8_2D_kernel
269
+ 37.545 ms vectorized_elementwise_kernel
270
+ REDUCE 36.084 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
271
+ 35.700 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
272
+ 29.428 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
273
+ 27.711 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
274
+ 24.683 ms triton_poi_fused__to_copy_cat_0
275
+ REDUCE 21.207 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
276
+ 16.757 ms unrolled_elementwise_kernel
277
+ 15.098 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
278
+ 12.401 ms ncclDevKernel_AllGather_RING_LL
279
+ 9.470 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
280
+ 8.827 ms triton_red_fused__to_copy_mul_silu_sum_1
281
+ 8.198 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
282
+ 7.712 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
283
+ 7.661 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
284
+ 6.966 ms triton_poi_fused__to_copy_mul_silu_0
285
+ 5.443 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
286
+ 5.231 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
287
+ 5.172 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
288
+ 4.336 ms setGroupedGemmWgradArguments
289
+ 3.938 ms rmsnorm_bwd_tuned_kernel
290
+ 3.906 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
291
+ 3.874 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
292
+ 3.840 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
293
+ 3.523 ms CatArrayBatchedCopy_alignedK_contig
294
+ 3.470 ms reduce_kernel
295
+ 3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
296
+ 3.308 ms _scatter_gather_elementwise_kernel
297
+ 3.151 ms multi_tensor_swizzle_row_scaling_kernel
298
+ 2.842 ms fused_rope_backward_kernel
299
+ 2.754 ms multi_tensor_swizzle_col_scaling_kernel
300
+ 2.449 ms fused_rope_forward_kernel
301
+ 2.260 ms swizzle_col_scaling_kernel
302
+ 1.863 ms rmsnorm_fwd_tuned_kernel
303
+
304
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 128 (64/iter)
306
+ reduce_grad sum : 28.645 ms/iter
307
+ reduce_grad busy (union) : 28.645 ms/iter
308
+ ... overlapped with compute: 27.822 ms/iter ( 97.1 %)
309
+ ... exposed on critical path: 0.824 ms/iter ( 2.9 %)
310
+ other GPU work (union) : 406.587 ms/iter
311
+ traced wall : 445.207 ms/iter
312
+ --> time hidden by overlap : 27.822 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 216.806 ms device_kernel
316
+ 96.363 ms elementwise_kernel
317
+ 57.991 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
318
+ 48.231 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
319
+ 48.211 ms multi_tensor_apply_kernel
320
+ 46.817 ms cast_mxfp8_2D_kernel
321
+ 39.392 ms vectorized_elementwise_kernel
322
+ REDUCE 36.916 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
323
+ 34.520 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
324
+ 30.079 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
325
+ 27.861 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
326
+ 24.711 ms triton_poi_fused__to_copy_cat_0
327
+ REDUCE 21.035 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
328
+ 16.825 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
329
+ 16.555 ms unrolled_elementwise_kernel
330
+ 11.149 ms ncclDevKernel_AllGather_RING_LL
331
+ 9.399 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
332
+ 8.915 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
333
+ 8.827 ms triton_red_fused__to_copy_mul_silu_sum_1
334
+ 8.185 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
335
+ 7.675 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
336
+ 6.996 ms triton_poi_fused__to_copy_mul_silu_0
337
+ 6.641 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
338
+ 5.400 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
339
+ 5.195 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
340
+ 4.437 ms setGroupedGemmWgradArguments
341
+ 3.971 ms rmsnorm_bwd_tuned_kernel
342
+ 3.900 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
343
+ 3.887 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
344
+ 3.808 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
345
+ 3.565 ms CatArrayBatchedCopy_alignedK_contig
346
+ 3.457 ms reduce_kernel
347
+ 3.392 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
348
+ 3.307 ms _scatter_gather_elementwise_kernel
349
+ 3.132 ms multi_tensor_swizzle_row_scaling_kernel
350
+ 2.861 ms fused_rope_backward_kernel
351
+ 2.774 ms multi_tensor_swizzle_col_scaling_kernel
352
+ 2.460 ms fused_rope_forward_kernel
353
+ 2.241 ms swizzle_col_scaling_kernel
354
+ 1.855 ms rmsnorm_fwd_tuned_kernel
355
+
356
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 128 (64/iter)
358
+ reduce_grad sum : 28.975 ms/iter
359
+ reduce_grad busy (union) : 28.975 ms/iter
360
+ ... overlapped with compute: 28.160 ms/iter ( 97.2 %)
361
+ ... exposed on critical path: 0.815 ms/iter ( 2.8 %)
362
+ other GPU work (union) : 407.996 ms/iter
363
+ traced wall : 484.104 ms/iter
364
+ --> time hidden by overlap : 28.160 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 215.091 ms device_kernel
368
+ 96.345 ms elementwise_kernel
369
+ 58.110 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
370
+ 48.206 ms multi_tensor_apply_kernel
371
+ 47.300 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
372
+ 46.736 ms cast_mxfp8_2D_kernel
373
+ REDUCE 37.555 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
374
+ 37.336 ms vectorized_elementwise_kernel
375
+ 35.418 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
376
+ 30.342 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
377
+ 27.735 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
378
+ 24.703 ms triton_poi_fused__to_copy_cat_0
379
+ REDUCE 20.303 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
380
+ 17.240 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
381
+ 16.659 ms unrolled_elementwise_kernel
382
+ 12.084 ms ncclDevKernel_AllGather_RING_LL
383
+ 9.517 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
384
+ 8.825 ms triton_red_fused__to_copy_mul_silu_sum_1
385
+ 8.545 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
386
+ 8.140 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
387
+ 7.712 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
388
+ 6.955 ms triton_poi_fused__to_copy_mul_silu_0
389
+ 6.666 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
390
+ 5.466 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
391
+ 5.190 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
392
+ 4.314 ms setGroupedGemmWgradArguments
393
+ 3.959 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
394
+ 3.941 ms rmsnorm_bwd_tuned_kernel
395
+ 3.872 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
396
+ 3.827 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
397
+ 3.536 ms CatArrayBatchedCopy_alignedK_contig
398
+ 3.480 ms reduce_kernel
399
+ 3.376 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
400
+ 3.309 ms _scatter_gather_elementwise_kernel
401
+ 3.233 ms multi_tensor_swizzle_row_scaling_kernel
402
+ 2.835 ms fused_rope_backward_kernel
403
+ 2.750 ms multi_tensor_swizzle_col_scaling_kernel
404
+ 2.475 ms fused_rope_forward_kernel
405
+ 2.304 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0
406
+ 2.263 ms swizzle_col_scaling_kernel
407
+
408
+ === G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 128 (64/iter)
410
+ reduce_grad sum : 28.929 ms/iter
411
+ reduce_grad busy (union) : 28.929 ms/iter
412
+ ... overlapped with compute: 28.125 ms/iter ( 97.2 %)
413
+ ... exposed on critical path: 0.804 ms/iter ( 2.8 %)
414
+ other GPU work (union) : 407.284 ms/iter
415
+ traced wall : 444.803 ms/iter
416
+ --> time hidden by overlap : 28.125 ms/iter
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=28.752 ms/iter overlap=27.939 ms/iter (97.2%) exposed=0.813 ms/iter status=PASS
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=G2
3
+ family=tierA
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce
6
+ commit=2e68c8ad
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=3584
14
+ num_experts=32
15
+ topk=8
16
+ experts_per_rank=4
17
+ expert_params=44040192
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=16
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=0
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-2
33
+ router_pre_softmax=0
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-5
38
+ seq_length=4096
39
+ micro_batch_size=1
40
+ global_batch_size=16
41
+ num_microbatches=2
42
+ assignments_per_rank=32768
43
+ mean_tokens_per_expert=8192
44
+ slot_policy=matched
45
+ slots_per_rank=4
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=1
49
+ probe_mode=0
50
+ async_grad_reduce=1
51
+ branch_role=timing
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=0
57
+ train_iters=10
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=1
64
+ kernel_trace=1
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
2
+ kernels : 133157
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df838cb4bcff056f30b96b8d1814b40ba9ec10475e43966ba8ef786ac5ecbae2
3
+ size 11588634
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 17.206125,
4
+ "overlapped_ms_per_iteration": 16.46425,
5
+ "exposed_ms_per_iteration": 0.742,
6
+ "overlap_pct": 95.68830866915124,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 225.290 ms device_kernel
4
+ 175.772 ms elementwise_kernel
5
+ 60.753 ms cast_mxfp8_2D_kernel
6
+ 59.799 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
7
+ 50.565 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
8
+ 47.968 ms multi_tensor_apply_kernel
9
+ 47.645 ms vectorized_elementwise_kernel
10
+ 28.042 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
11
+ 26.365 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
12
+ 24.993 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
13
+ 24.753 ms triton_poi_fused__to_copy_cat_0
14
+ 24.705 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
15
+ REDUCE 23.330 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
16
+ 16.295 ms unrolled_elementwise_kernel
17
+ 13.756 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
18
+ REDUCE 11.254 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
19
+ 10.528 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
20
+ 8.926 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
21
+ 8.655 ms ncclDevKernel_AllGather_RING_LL
22
+ 8.277 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
23
+ 7.837 ms triton_red_fused__to_copy_mul_silu_sum_1
24
+ 7.778 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
25
+ 6.925 ms triton_poi_fused__to_copy_mul_silu_0
26
+ 6.235 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
27
+ 5.168 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
28
+ 4.980 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
29
+ 4.198 ms multi_tensor_swizzle_row_scaling_kernel
30
+ 4.187 ms setGroupedGemmWgradArguments
31
+ 4.143 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
32
+ 3.938 ms _scatter_gather_elementwise_kernel
33
+ 3.926 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
34
+ 3.922 ms rmsnorm_bwd_tuned_kernel
35
+ 3.805 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
36
+ 3.629 ms reduce_kernel
37
+ 3.471 ms CatArrayBatchedCopy_alignedK_contig
38
+ 3.388 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
39
+ 3.004 ms gatherTopK
40
+ 2.833 ms multi_tensor_swizzle_col_scaling_kernel
41
+ 2.621 ms fused_rope_backward_kernel
42
+ 2.455 ms fused_rope_forward_kernel
43
+
44
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 128 (64/iter)
46
+ reduce_grad sum : 17.292 ms/iter
47
+ reduce_grad busy (union) : 17.292 ms/iter
48
+ ... overlapped with compute: 16.580 ms/iter ( 95.9 %)
49
+ ... exposed on critical path: 0.712 ms/iter ( 4.1 %)
50
+ other GPU work (union) : 469.865 ms/iter
51
+ traced wall : 660.128 ms/iter
52
+ --> time hidden by overlap : 16.580 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 224.560 ms device_kernel
56
+ 180.765 ms elementwise_kernel
57
+ 60.594 ms cast_mxfp8_2D_kernel
58
+ 59.763 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
59
+ 48.869 ms vectorized_elementwise_kernel
60
+ 48.758 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
61
+ 48.204 ms multi_tensor_apply_kernel
62
+ 25.708 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
63
+ 25.182 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
64
+ 24.859 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
65
+ 24.786 ms triton_poi_fused__to_copy_cat_0
66
+ REDUCE 22.963 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
67
+ 17.737 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
68
+ 16.598 ms unrolled_elementwise_kernel
69
+ 13.843 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
70
+ 11.722 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
71
+ REDUCE 11.356 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
72
+ 9.512 ms ncclDevKernel_AllGather_RING_LL
73
+ 9.031 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
74
+ 8.225 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
75
+ 7.820 ms triton_red_fused__to_copy_mul_silu_sum_1
76
+ 7.719 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
77
+ 7.014 ms triton_poi_fused__to_copy_mul_silu_0
78
+ 6.119 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
79
+ 5.191 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
80
+ 4.987 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
81
+ 4.302 ms multi_tensor_swizzle_row_scaling_kernel
82
+ 4.283 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
83
+ 4.261 ms setGroupedGemmWgradArguments
84
+ 3.956 ms _scatter_gather_elementwise_kernel
85
+ 3.931 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
86
+ 3.931 ms rmsnorm_bwd_tuned_kernel
87
+ 3.800 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
88
+ 3.594 ms reduce_kernel
89
+ 3.456 ms CatArrayBatchedCopy_alignedK_contig
90
+ 3.370 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
91
+ 2.998 ms gatherTopK
92
+ 2.871 ms multi_tensor_swizzle_col_scaling_kernel
93
+ 2.626 ms fused_rope_backward_kernel
94
+ 2.447 ms fused_rope_forward_kernel
95
+
96
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 128 (64/iter)
98
+ reduce_grad sum : 17.159 ms/iter
99
+ reduce_grad busy (union) : 17.159 ms/iter
100
+ ... overlapped with compute: 16.403 ms/iter ( 95.6 %)
101
+ ... exposed on critical path: 0.756 ms/iter ( 4.4 %)
102
+ other GPU work (union) : 468.435 ms/iter
103
+ traced wall : 630.040 ms/iter
104
+ --> time hidden by overlap : 16.403 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 222.602 ms device_kernel
108
+ 180.628 ms elementwise_kernel
109
+ 60.950 ms cast_mxfp8_2D_kernel
110
+ 59.707 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
111
+ 49.101 ms vectorized_elementwise_kernel
112
+ 48.859 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
113
+ 48.325 ms multi_tensor_apply_kernel
114
+ 25.365 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
115
+ 25.357 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
116
+ 24.947 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
117
+ 24.729 ms triton_poi_fused__to_copy_cat_0
118
+ REDUCE 22.867 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
119
+ 17.862 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
120
+ 16.723 ms unrolled_elementwise_kernel
121
+ 13.896 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
122
+ 12.801 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
123
+ REDUCE 11.352 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
124
+ 10.792 ms ncclDevKernel_AllGather_RING_LL
125
+ 9.070 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
126
+ 8.304 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
127
+ 7.849 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
128
+ 7.822 ms triton_red_fused__to_copy_mul_silu_sum_1
129
+ 6.960 ms triton_poi_fused__to_copy_mul_silu_0
130
+ 6.097 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
131
+ 5.278 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
132
+ 5.067 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
133
+ 4.291 ms multi_tensor_swizzle_row_scaling_kernel
134
+ 4.283 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
135
+ 4.184 ms setGroupedGemmWgradArguments
136
+ 3.992 ms rmsnorm_bwd_tuned_kernel
137
+ 3.987 ms _scatter_gather_elementwise_kernel
138
+ 3.855 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
139
+ 3.843 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
140
+ 3.624 ms reduce_kernel
141
+ 3.466 ms CatArrayBatchedCopy_alignedK_contig
142
+ 3.373 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
143
+ 3.008 ms gatherTopK
144
+ 2.815 ms multi_tensor_swizzle_col_scaling_kernel
145
+ 2.626 ms fused_rope_backward_kernel
146
+ 2.457 ms fused_rope_forward_kernel
147
+
148
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 128 (64/iter)
150
+ reduce_grad sum : 17.110 ms/iter
151
+ reduce_grad busy (union) : 17.110 ms/iter
152
+ ... overlapped with compute: 16.374 ms/iter ( 95.7 %)
153
+ ... exposed on critical path: 0.736 ms/iter ( 4.3 %)
154
+ other GPU work (union) : 469.923 ms/iter
155
+ traced wall : 660.796 ms/iter
156
+ --> time hidden by overlap : 16.374 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 228.029 ms device_kernel
160
+ 180.601 ms elementwise_kernel
161
+ 60.977 ms cast_mxfp8_2D_kernel
162
+ 59.761 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
163
+ 49.678 ms vectorized_elementwise_kernel
164
+ 49.237 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
165
+ 48.234 ms multi_tensor_apply_kernel
166
+ 25.169 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
167
+ 25.044 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
168
+ 24.896 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
169
+ 24.787 ms triton_poi_fused__to_copy_cat_0
170
+ REDUCE 22.655 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
171
+ 16.728 ms unrolled_elementwise_kernel
172
+ 13.867 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
173
+ 12.902 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
174
+ 11.896 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
175
+ REDUCE 11.669 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
176
+ 8.988 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
177
+ 8.917 ms ncclDevKernel_AllGather_RING_LL
178
+ 8.285 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
179
+ 7.892 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
180
+ 7.820 ms triton_red_fused__to_copy_mul_silu_sum_1
181
+ 6.999 ms triton_poi_fused__to_copy_mul_silu_0
182
+ 5.768 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
183
+ 5.216 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
184
+ 5.026 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
185
+ 4.285 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
186
+ 4.278 ms multi_tensor_swizzle_row_scaling_kernel
187
+ 4.145 ms setGroupedGemmWgradArguments
188
+ 3.997 ms _scatter_gather_elementwise_kernel
189
+ 3.968 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
190
+ 3.948 ms rmsnorm_bwd_tuned_kernel
191
+ 3.840 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
192
+ 3.675 ms reduce_kernel
193
+ 3.465 ms CatArrayBatchedCopy_alignedK_contig
194
+ 3.439 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
195
+ 3.011 ms gatherTopK
196
+ 2.870 ms multi_tensor_swizzle_col_scaling_kernel
197
+ 2.625 ms fused_rope_backward_kernel
198
+ 2.449 ms fused_rope_forward_kernel
199
+
200
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 128 (64/iter)
202
+ reduce_grad sum : 17.162 ms/iter
203
+ reduce_grad busy (union) : 17.162 ms/iter
204
+ ... overlapped with compute: 16.412 ms/iter ( 95.6 %)
205
+ ... exposed on critical path: 0.750 ms/iter ( 4.4 %)
206
+ other GPU work (union) : 468.637 ms/iter
207
+ traced wall : 634.045 ms/iter
208
+ --> time hidden by overlap : 16.412 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 228.089 ms device_kernel
212
+ 180.813 ms elementwise_kernel
213
+ 60.956 ms cast_mxfp8_2D_kernel
214
+ 59.664 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
215
+ 48.814 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
216
+ 48.233 ms multi_tensor_apply_kernel
217
+ 47.464 ms vectorized_elementwise_kernel
218
+ 25.377 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
219
+ 25.019 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
220
+ 24.960 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
221
+ 24.804 ms triton_poi_fused__to_copy_cat_0
222
+ REDUCE 22.213 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
223
+ 16.756 ms unrolled_elementwise_kernel
224
+ 15.607 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
225
+ 13.913 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
226
+ REDUCE 12.357 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
227
+ 11.126 ms ncclDevKernel_AllGather_RING_LL
228
+ 9.766 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
229
+ 9.194 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
230
+ 8.316 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
231
+ 7.846 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
232
+ 7.817 ms triton_red_fused__to_copy_mul_silu_sum_1
233
+ 7.017 ms triton_poi_fused__to_copy_mul_silu_0
234
+ 5.302 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
235
+ 5.110 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
236
+ 5.063 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
237
+ 4.289 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
238
+ 4.220 ms setGroupedGemmWgradArguments
239
+ 4.210 ms multi_tensor_swizzle_row_scaling_kernel
240
+ 4.011 ms _scatter_gather_elementwise_kernel
241
+ 3.989 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
242
+ 3.984 ms rmsnorm_bwd_tuned_kernel
243
+ 3.853 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
244
+ 3.658 ms reduce_kernel
245
+ 3.472 ms CatArrayBatchedCopy_alignedK_contig
246
+ 3.423 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
247
+ 3.014 ms gatherTopK
248
+ 2.881 ms multi_tensor_swizzle_col_scaling_kernel
249
+ 2.629 ms fused_rope_backward_kernel
250
+ 2.463 ms fused_rope_forward_kernel
251
+
252
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 128 (64/iter)
254
+ reduce_grad sum : 17.285 ms/iter
255
+ reduce_grad busy (union) : 17.285 ms/iter
256
+ ... overlapped with compute: 16.549 ms/iter ( 95.7 %)
257
+ ... exposed on critical path: 0.736 ms/iter ( 4.3 %)
258
+ other GPU work (union) : 468.284 ms/iter
259
+ traced wall : 584.115 ms/iter
260
+ --> time hidden by overlap : 16.549 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 223.409 ms device_kernel
264
+ 180.662 ms elementwise_kernel
265
+ 60.780 ms cast_mxfp8_2D_kernel
266
+ 59.720 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
267
+ 49.220 ms vectorized_elementwise_kernel
268
+ 49.112 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
269
+ 48.218 ms multi_tensor_apply_kernel
270
+ 25.591 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
271
+ 25.158 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
272
+ 24.895 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
273
+ 24.739 ms triton_poi_fused__to_copy_cat_0
274
+ REDUCE 22.718 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
275
+ 18.214 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
276
+ 16.703 ms unrolled_elementwise_kernel
277
+ 13.817 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
278
+ 11.849 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
279
+ REDUCE 11.639 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
280
+ 10.266 ms ncclDevKernel_AllGather_RING_LL
281
+ 9.080 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
282
+ 8.270 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
283
+ 7.821 ms triton_red_fused__to_copy_mul_silu_sum_1
284
+ 7.752 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
285
+ 6.980 ms triton_poi_fused__to_copy_mul_silu_0
286
+ 5.730 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
287
+ 5.234 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
288
+ 5.087 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
289
+ 4.290 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
290
+ 4.239 ms multi_tensor_swizzle_row_scaling_kernel
291
+ 4.198 ms setGroupedGemmWgradArguments
292
+ 3.993 ms _scatter_gather_elementwise_kernel
293
+ 3.956 ms rmsnorm_bwd_tuned_kernel
294
+ 3.875 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
295
+ 3.832 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
296
+ 3.620 ms reduce_kernel
297
+ 3.457 ms CatArrayBatchedCopy_alignedK_contig
298
+ 3.374 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
299
+ 3.020 ms gatherTopK
300
+ 2.833 ms multi_tensor_swizzle_col_scaling_kernel
301
+ 2.630 ms fused_rope_backward_kernel
302
+ 2.447 ms fused_rope_forward_kernel
303
+
304
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 128 (64/iter)
306
+ reduce_grad sum : 17.178 ms/iter
307
+ reduce_grad busy (union) : 17.178 ms/iter
308
+ ... overlapped with compute: 16.423 ms/iter ( 95.6 %)
309
+ ... exposed on critical path: 0.755 ms/iter ( 4.4 %)
310
+ other GPU work (union) : 469.125 ms/iter
311
+ traced wall : 659.490 ms/iter
312
+ --> time hidden by overlap : 16.423 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 225.278 ms device_kernel
316
+ 180.770 ms elementwise_kernel
317
+ 60.786 ms cast_mxfp8_2D_kernel
318
+ 59.662 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
319
+ 48.635 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
320
+ 48.231 ms multi_tensor_apply_kernel
321
+ 46.978 ms vectorized_elementwise_kernel
322
+ 25.916 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
323
+ 25.417 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
324
+ 24.999 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
325
+ 24.748 ms triton_poi_fused__to_copy_cat_0
326
+ REDUCE 22.619 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
327
+ 18.939 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
328
+ 16.603 ms unrolled_elementwise_kernel
329
+ 13.962 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
330
+ REDUCE 11.866 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
331
+ 10.605 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
332
+ 10.150 ms ncclDevKernel_AllGather_RING_LL
333
+ 8.950 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
334
+ 8.258 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
335
+ 7.821 ms triton_red_fused__to_copy_mul_silu_sum_1
336
+ 7.742 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
337
+ 6.993 ms triton_poi_fused__to_copy_mul_silu_0
338
+ 5.393 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
339
+ 5.175 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
340
+ 4.976 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
341
+ 4.286 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
342
+ 4.151 ms multi_tensor_swizzle_row_scaling_kernel
343
+ 4.109 ms setGroupedGemmWgradArguments
344
+ 3.962 ms _scatter_gather_elementwise_kernel
345
+ 3.950 ms rmsnorm_bwd_tuned_kernel
346
+ 3.888 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
347
+ 3.808 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
348
+ 3.621 ms reduce_kernel
349
+ 3.454 ms CatArrayBatchedCopy_alignedK_contig
350
+ 3.386 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
351
+ 3.005 ms gatherTopK
352
+ 2.846 ms multi_tensor_swizzle_col_scaling_kernel
353
+ 2.631 ms fused_rope_backward_kernel
354
+ 2.452 ms fused_rope_forward_kernel
355
+
356
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 128 (64/iter)
358
+ reduce_grad sum : 17.242 ms/iter
359
+ reduce_grad busy (union) : 17.242 ms/iter
360
+ ... overlapped with compute: 16.477 ms/iter ( 95.6 %)
361
+ ... exposed on critical path: 0.765 ms/iter ( 4.4 %)
362
+ other GPU work (union) : 468.131 ms/iter
363
+ traced wall : 583.652 ms/iter
364
+ --> time hidden by overlap : 16.477 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 223.053 ms device_kernel
368
+ 180.620 ms elementwise_kernel
369
+ 60.854 ms cast_mxfp8_2D_kernel
370
+ 59.672 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
371
+ 48.698 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
372
+ 48.292 ms multi_tensor_apply_kernel
373
+ 47.209 ms vectorized_elementwise_kernel
374
+ 25.902 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
375
+ 25.368 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0
376
+ 24.804 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
377
+ 24.734 ms triton_poi_fused__to_copy_cat_0
378
+ REDUCE 22.758 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
379
+ 20.086 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
380
+ 16.769 ms unrolled_elementwise_kernel
381
+ 13.922 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0
382
+ REDUCE 11.685 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
383
+ 10.928 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
384
+ 8.968 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
385
+ 8.420 ms ncclDevKernel_AllGather_RING_LL
386
+ 8.248 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
387
+ 7.820 ms triton_red_fused__to_copy_mul_silu_sum_1
388
+ 7.793 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
389
+ 6.966 ms triton_poi_fused__to_copy_mul_silu_0
390
+ 6.195 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0
391
+ 5.234 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
392
+ 5.026 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
393
+ 4.274 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
394
+ 4.267 ms multi_tensor_swizzle_row_scaling_kernel
395
+ 4.140 ms setGroupedGemmWgradArguments
396
+ 4.015 ms _scatter_gather_elementwise_kernel
397
+ 3.906 ms rmsnorm_bwd_tuned_kernel
398
+ 3.865 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
399
+ 3.837 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
400
+ 3.630 ms reduce_kernel
401
+ 3.469 ms CatArrayBatchedCopy_alignedK_contig
402
+ 3.366 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
403
+ 3.016 ms gatherTopK
404
+ 2.825 ms multi_tensor_swizzle_col_scaling_kernel
405
+ 2.627 ms fused_rope_backward_kernel
406
+ 2.452 ms fused_rope_forward_kernel
407
+
408
+ === G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 128 (64/iter)
410
+ reduce_grad sum : 17.221 ms/iter
411
+ reduce_grad busy (union) : 17.221 ms/iter
412
+ ... overlapped with compute: 16.496 ms/iter ( 95.8 %)
413
+ ... exposed on critical path: 0.726 ms/iter ( 4.2 %)
414
+ other GPU work (union) : 467.542 ms/iter
415
+ traced wall : 582.845 ms/iter
416
+ --> time hidden by overlap : 16.496 ms/iter
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=17.206 ms/iter overlap=16.464 ms/iter (95.7%) exposed=0.742 ms/iter status=PASS
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=G3
3
+ family=tierA
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce
6
+ commit=2e68c8ad
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=1792
14
+ num_experts=64
15
+ topk=16
16
+ experts_per_rank=8
17
+ expert_params=22020096
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=16
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=0
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-2
33
+ router_pre_softmax=0
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-5
38
+ seq_length=4096
39
+ micro_batch_size=1
40
+ global_batch_size=16
41
+ num_microbatches=2
42
+ assignments_per_rank=65536
43
+ mean_tokens_per_expert=8192
44
+ slot_policy=matched
45
+ slots_per_rank=8
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=1
49
+ probe_mode=0
50
+ async_grad_reduce=1
51
+ branch_role=timing
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=0
57
+ train_iters=10
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=1
64
+ kernel_trace=1
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
2
+ kernels : 141526
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=G4
3
+ family=tierA
4
+ repo=/workspace/Megatron-MoE-Echo
5
+ branch=moonep_backend_async_grad_reduce
6
+ commit=2e68c8ad
7
+ moonep_source_dir=/workspace/MoonEP
8
+ moonep_source_branch=megatron-ddp-symmetric-params
9
+ moonep_source_commit=d8bc666
10
+ num_layers=16
11
+ hidden=4096
12
+ ffn_hidden=14336
13
+ moe_ffn_hidden=896
14
+ num_experts=128
15
+ topk=32
16
+ experts_per_rank=16
17
+ expert_params=11010048
18
+ heads=32
19
+ kv_groups=8
20
+ kv_channels=
21
+ qk_layernorm=0
22
+ shared_ffn=0
23
+ shared_expert_overlap=0
24
+ moe_token_dispatcher=flex
25
+ mla=0
26
+ dense_first_layers=0
27
+ moe_layers=16
28
+ rotary_base=1000000
29
+ rotary_scaling=
30
+ qkv_bias=0
31
+ router_balancing=aux_loss
32
+ aux_loss_coeff=1e-2
33
+ router_pre_softmax=0
34
+ router_fusion=0
35
+ router_dtype=fp32
36
+ device_initiated_grouped_gemm=1
37
+ norm_epsilon=1e-5
38
+ seq_length=4096
39
+ micro_batch_size=1
40
+ global_batch_size=16
41
+ num_microbatches=2
42
+ assignments_per_rank=131072
43
+ mean_tokens_per_expert=8192
44
+ slot_policy=matched
45
+ slots_per_rank=16
46
+ capacity_mode=none
47
+ capacity=none
48
+ cuda_graph=1
49
+ probe_mode=0
50
+ async_grad_reduce=1
51
+ branch_role=timing
52
+ memory_dump_mode=none
53
+ memory_snapshot_path=
54
+ cuda_graph_warmup_steps=3
55
+ recompute=none
56
+ force_balance=0
57
+ train_iters=10
58
+ ep_size=8
59
+ echo_dispatch_overlap=1
60
+ echo_dispatch_recompute=0
61
+ ultraep_quota_min_tokens=
62
+ ultraep_weight_sync_join=
63
+ profile_enable=1
64
+ kernel_trace=1
65
+ fp8_mode=mxfp8
66
+ fp8_recipe=mxfp8
67
+ training_seed=1234
68
+ learning_rate=1e-4
69
+ min_learning_rate=1.0e-5
70
+ lr_warmup_iters=5
71
+ state_dump_enable=0
72
+ state_dump_dir=
73
+ moonep_assert_no_pending_reduce=1
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log ADDED
@@ -0,0 +1,1382 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ==============================================================
2
+ G4 / moonep (slots/rank=16 policy=matched)
3
+ E=128 K=32 H'=896 m=16
4
+ L=16 seq=4096 gbs=16 (2 microbatches)
5
+ cap_mode=none cap=none cg=1 recompute=none
6
+ memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/memory_snapshots/snapshot.pickle
7
+ repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad
8
+ ==============================================================
9
+ W0830 09:43:03.075000 944198 torch/distributed/run.py:766]
10
+ W0830 09:43:03.075000 944198 torch/distributed/run.py:766] *****************************************
11
+ W0830 09:43:03.075000 944198 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
12
+ W0830 09:43:03.075000 944198 torch/distributed/run.py:766] *****************************************
13
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
14
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
15
+ using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
16
+ WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer
17
+ Number of virtual stages per pipeline stage: None
18
+ accumulate and all-reduce gradients in fp32 for bfloat16 data type.
19
+ using torch.bfloat16 for parameters ...
20
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
21
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
22
+ ------------------------ arguments ------------------------
23
+ account_for_embedding_in_pipeline_split ......... False
24
+ account_for_loss_in_pipeline_split .............. False
25
+ accumulate_allreduce_grads_in_fp32 .............. True
26
+ adam_beta1 ...................................... 0.9
27
+ adam_beta2 ...................................... 0.999
28
+ adam_eps ........................................ 1e-08
29
+ add_bias_linear ................................. False
30
+ add_position_embedding .......................... False
31
+ add_qkv_bias .................................... False
32
+ adlr_autoresume ................................. False
33
+ adlr_autoresume_interval ........................ 1000
34
+ align_grad_reduce ............................... True
35
+ align_param_gather .............................. False
36
+ app_tag_run_name ................................ None
37
+ app_tag_run_version ............................. 0.0.0
38
+ apply_layernorm_1p .............................. False
39
+ apply_query_key_layer_scaling ................... False
40
+ apply_residual_connection_post_layernorm ........ False
41
+ apply_rope_fusion ............................... True
42
+ async_save ...................................... None
43
+ async_tensor_model_parallel_allreduce ........... True
44
+ attention_backend ............................... AttnBackend.auto
45
+ attention_dropout ............................... 0.0
46
+ attention_softmax_in_fp32 ....................... False
47
+ auto_detect_ckpt_format ......................... False
48
+ barrier_with_L1_time ............................ True
49
+ bert_binary_head ................................ True
50
+ bert_embedder_type .............................. megatron
51
+ bert_load ....................................... None
52
+ bf16 ............................................ True
53
+ bias_dropout_fusion ............................. True
54
+ bias_gelu_fusion ................................ False
55
+ bias_swiglu_fusion .............................. True
56
+ biencoder_projection_dim ........................ 0
57
+ biencoder_shared_query_context_model ............ False
58
+ block_data_path ................................. None
59
+ calc_ft_timeouts ................................ False
60
+ calculate_per_token_loss ........................ False
61
+ check_for_large_grads ........................... False
62
+ check_for_nan_in_loss_and_grad .................. False
63
+ check_for_spiky_loss ............................ False
64
+ check_weight_hash_across_dp_replicas_interval ... None
65
+ ckpt_assume_constant_structure .................. False
66
+ ckpt_convert_format ............................. None
67
+ ckpt_convert_save ............................... None
68
+ ckpt_convert_update_legacy_dist_opt_format ...... False
69
+ ckpt_format ..................................... torch_dist
70
+ ckpt_fully_parallel_load ........................ False
71
+ ckpt_fully_parallel_save ........................ True
72
+ ckpt_fully_parallel_save_deprecated ............. False
73
+ ckpt_step ....................................... None
74
+ classes_fraction ................................ 1.0
75
+ clip_grad ....................................... 1.0
76
+ clone_scatter_output_in_embedding ............... True
77
+ config_logger_dir ...............................
78
+ consumed_train_samples .......................... 0
79
+ consumed_valid_samples .......................... 0
80
+ context_parallel_size ........................... 1
81
+ cp_comm_type .................................... ['p2p']
82
+ create_attention_mask_in_dataloader ............. True
83
+ cross_entropy_fusion_impl ....................... native
84
+ cross_entropy_loss_fusion ....................... False
85
+ cuda_graph_scope ................................ full_iteration
86
+ cuda_graph_warmup_steps ......................... 3
87
+ data_args_path .................................. None
88
+ data_cache_path ................................. None
89
+ data_parallel_random_init ....................... False
90
+ data_parallel_sharding_strategy ................. no_shard
91
+ data_parallel_size .............................. 8
92
+ data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document']
93
+ data_per_class_fraction ......................... 1.0
94
+ data_sharding ................................... True
95
+ dataloader_type ................................. single
96
+ ddp_average_in_collective ....................... False
97
+ ddp_bucket_size ................................. None
98
+ ddp_num_buckets ................................. None
99
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
100
+ decoder_first_pipeline_num_layers ............... None
101
+ decoder_last_pipeline_num_layers ................ None
102
+ decoder_num_layers .............................. None
103
+ decoder_seq_length .............................. None
104
+ decoupled_lr .................................... None
105
+ decoupled_min_lr ................................ None
106
+ decrease_batch_size_if_needed ................... False
107
+ defer_embedding_wgrad_compute ................... False
108
+ delay_wgrad_compute ............................. False
109
+ deprecated_use_mcore_models ..................... True
110
+ deterministic_mode .............................. False
111
+ dino_bottleneck_size ............................ 256
112
+ dino_freeze_last_layer .......................... 1
113
+ dino_head_hidden_size ........................... 2048
114
+ dino_local_crops_number ......................... 10
115
+ dino_local_img_size ............................. 96
116
+ dino_norm_last_layer ............................ False
117
+ dino_teacher_temp ............................... 0.07
118
+ dino_warmup_teacher_temp ........................ 0.04
119
+ dino_warmup_teacher_temp_epochs ................. 30
120
+ disable_bf16_reduced_precision_matmul ........... False
121
+ disable_mamba_mem_eff_path ...................... False
122
+ disable_straggler_on_startup .................... False
123
+ dist_ckpt_format_deprecated ..................... None
124
+ dist_ckpt_strictness ............................ assume_ok_unexpected
125
+ distribute_saved_activations .................... False
126
+ distributed_backend ............................. nccl
127
+ distributed_timeout_minutes ..................... 10
128
+ embedding_init_method_std ....................... None
129
+ embedding_path .................................. None
130
+ empty_unused_memory_level ....................... 0
131
+ enable_cuda_graph ............................... True
132
+ enable_experimental ............................. False
133
+ enable_ft_package ............................... False
134
+ enable_gloo_process_groups ...................... True
135
+ enable_msc ...................................... True
136
+ enable_one_logger ............................... True
137
+ encoder_num_layers .............................. 16
138
+ encoder_seq_length .............................. 4096
139
+ end_weight_decay ................................ 0.1
140
+ eod_mask_loss ................................... False
141
+ error_injection_rate ............................ 0
142
+ error_injection_type ............................ transient_error
143
+ eval_interval ................................... 1000000
144
+ eval_iters ...................................... 0
145
+ evidence_data_path .............................. None
146
+ exit_duration_in_mins ........................... None
147
+ exit_interval ................................... None
148
+ exit_on_missing_checkpoint ...................... False
149
+ exit_signal_handler ............................. False
150
+ exp_avg_dtype ................................... torch.float32
151
+ exp_avg_sq_dtype ................................ torch.float32
152
+ expert_model_parallel_size ...................... 8
153
+ expert_tensor_parallel_size ..................... 1
154
+ export_force_local_attention .................... False
155
+ export_kd_cfg ................................... None
156
+ export_kd_teacher_ckpt_format ................... None
157
+ export_kd_teacher_load .......................... None
158
+ export_kv_cache_quant ........................... False
159
+ export_legacy_megatron .......................... False
160
+ export_model_type ............................... GPTModel
161
+ export_moe_apply_probs_on_input ................. False
162
+ export_qk_l2_norm ............................... False
163
+ export_quant_cfg ................................ None
164
+ export_real_quant_cfg ........................... None
165
+ export_te_mcore_model ........................... False
166
+ external_cuda_graph ............................. False
167
+ ffn_hidden_size ................................. 14336
168
+ finetune ........................................ False
169
+ finetune_data_split ............................. train
170
+ finetune_hf_dataset ............................. None
171
+ first_last_layers_bf16 .......................... False
172
+ flash_decode .................................... False
173
+ fp16 ............................................ False
174
+ fp16_lm_cross_entropy ........................... False
175
+ fp32_residual_connection ........................ False
176
+ fp8 ............................................. e4m3
177
+ fp8_amax_compute_algo ........................... most_recent
178
+ fp8_amax_history_len ............................ 1
179
+ fp8_interval .................................... 1
180
+ fp8_margin ...................................... 0
181
+ fp8_param_gather ................................ False
182
+ fp8_recipe ...................................... mxfp8
183
+ fp8_wgrad ....................................... True
184
+ fsdp_double_buffer .............................. False
185
+ global_batch_size ............................... 16
186
+ grad_reduce_in_bf16 ............................. False
187
+ gradient_accumulation_fusion .................... True
188
+ gradient_reduce_div_fusion ...................... True
189
+ group_query_attention ........................... True
190
+ head_lr_mult .................................... 1.0
191
+ heterogeneous_layers_config_encoded_json ........ None
192
+ heterogeneous_layers_config_path ................ None
193
+ hidden_dropout .................................. 0.0
194
+ hidden_size ..................................... 4096
195
+ hierarchical_context_parallel_sizes ............. None
196
+ high_priority_stream_groups ..................... []
197
+ hybrid_attention_ratio .......................... 0.0
198
+ hybrid_mlp_ratio ................................ 0.0
199
+ hybrid_override_pattern ......................... None
200
+ hysteresis ...................................... 2
201
+ ict_head_size ................................... None
202
+ ict_load ........................................ None
203
+ img_h ........................................... 224
204
+ img_w ........................................... 224
205
+ indexer_batch_size .............................. 128
206
+ indexer_log_interval ............................ 1000
207
+ inference_batch_times_seqlen_threshold .......... -1
208
+ inference_dynamic_batching ...................... False
209
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
210
+ inference_dynamic_batching_buffer_overflow_factor None
211
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
212
+ inference_dynamic_batching_chunk_size ........... 256
213
+ inference_dynamic_batching_max_requests_override None
214
+ inference_dynamic_batching_max_tokens_override .. None
215
+ inference_dynamic_batching_num_cuda_graphs ...... 16
216
+ inference_max_batch_size ........................ 8
217
+ inference_max_seq_length ........................ 2560
218
+ inference_rng_tracker ........................... False
219
+ init_method_std ................................. 0.01
220
+ init_method_xavier_uniform ...................... False
221
+ init_model_with_meta_device ..................... False
222
+ initial_loss_scale .............................. 4294967296
223
+ inprocess_active_world_size ..................... 8
224
+ inprocess_barrier_timeout ....................... 120
225
+ inprocess_completion_timeout .................... 120
226
+ inprocess_empty_cuda_cache ...................... False
227
+ inprocess_granularity ........................... node
228
+ inprocess_hard_timeout .......................... 90
229
+ inprocess_heartbeat_interval .................... 30
230
+ inprocess_heartbeat_timeout ..................... 60
231
+ inprocess_last_call_wait ........................ 1
232
+ inprocess_max_iterations ........................ None
233
+ inprocess_monitor_process_interval .............. 1.0
234
+ inprocess_monitor_thread_interval ............... 1.0
235
+ inprocess_progress_watchdog_interval ............ 1.0
236
+ inprocess_restart ............................... False
237
+ inprocess_soft_timeout .......................... 60
238
+ inprocess_termination_grace_time ................ 1
239
+ is_hybrid_model ................................. False
240
+ iter_per_epoch .................................. 1250
241
+ iterations_to_skip .............................. []
242
+ keep_fp8_transpose_cache_when_using_custom_fsdp . False
243
+ kitchen_config_file ............................. None
244
+ kitchen_recipe_number ........................... None
245
+ kv_channels ..................................... 128
246
+ kv_lora_rank .................................... 32
247
+ lazy_mpu_init ................................... None
248
+ load ............................................ None
249
+ load_main_params_from_ckpt ...................... None
250
+ load_model_opt_format ........................... False
251
+ local_rank ...................................... 0
252
+ log_energy ...................................... False
253
+ log_interval .................................... 1
254
+ log_loss_scale_to_tensorboard ................... True
255
+ log_memory_to_tensorboard ....................... False
256
+ log_num_zeros_in_grad ........................... False
257
+ log_params_norm ................................. False
258
+ log_progress .................................... False
259
+ log_straggler ................................... False
260
+ log_throughput .................................. True
261
+ log_timers_to_tensorboard ....................... False
262
+ log_validation_ppl_to_tensorboard ............... False
263
+ log_world_size_to_tensorboard ................... False
264
+ logging_level ................................... None
265
+ loss_scale ...................................... None
266
+ loss_scale_window ............................... 1000
267
+ lr .............................................. 0.0001
268
+ lr_decay_iters .................................. 8
269
+ lr_decay_samples ................................ None
270
+ lr_decay_style .................................. cosine
271
+ lr_warmup_fraction .............................. None
272
+ lr_warmup_init .................................. 0.0
273
+ lr_warmup_iters ................................. 5
274
+ lr_warmup_samples ............................... 0
275
+ lr_wsd_decay_iters .............................. None
276
+ lr_wsd_decay_samples ............................ None
277
+ lr_wsd_decay_style .............................. exponential
278
+ main_grads_dtype ................................ torch.float32
279
+ main_params_dtype ............................... torch.float32
280
+ make_vocab_size_divisible_by .................... 128
281
+ mamba_head_dim .................................. 64
282
+ mamba_num_groups ................................ 8
283
+ mamba_num_heads ................................. None
284
+ mamba_state_dim ................................. 128
285
+ manual_gc ....................................... False
286
+ manual_gc_eval .................................. True
287
+ manual_gc_interval .............................. 0
288
+ mask_factor ..................................... 1.0
289
+ mask_prob ....................................... 0.15
290
+ mask_type ....................................... random
291
+ masked_softmax_fusion ........................... False
292
+ max_position_embeddings ......................... 32768
293
+ max_tokens_to_oom ............................... 12000
294
+ memory_snapshot_path ............................ snapshot.pickle
295
+ merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt
296
+ micro_batch_size ................................ 1
297
+ microbatch_group_size_per_vp_stage .............. None
298
+ mid_level_dataset_surplus ....................... 0.005
299
+ min_loss_scale .................................. 1.0
300
+ min_lr .......................................... 1e-05
301
+ mlp_chunks_for_prefill .......................... 1
302
+ mmap_bin_files .................................. True
303
+ mock_data ....................................... False
304
+ moe_apply_probs_on_input ........................ False
305
+ moe_aux_loss_coeff .............................. 0.01
306
+ moe_deepep_num_sms .............................. 20
307
+ moe_echo_enable_random_offloading ............... False
308
+ moe_echo_expert_dispatch_overlap ................ False
309
+ moe_echo_expert_dispatcher_type ................. hybridep
310
+ moe_echo_recompute_expert_dispatch .............. False
311
+ moe_enable_deepep ............................... False
312
+ moe_enable_echo ................................. False
313
+ moe_enable_hybridep ............................. False
314
+ moe_enable_moonep ............................... True
315
+ moe_expert_capacity_factor ...................... None
316
+ moe_extended_tp ................................. False
317
+ moe_ffn_hidden_size ............................. 896
318
+ moe_grouped_gemm ................................ True
319
+ moe_hybridep_num_sms ............................ 16
320
+ moe_input_jitter_eps ............................ None
321
+ moe_layer_freq .................................. 1
322
+ moe_layer_recompute ............................. False
323
+ moe_moonep_async_grad_reduce .................... True
324
+ moe_moonep_num_sms .............................. None
325
+ moe_num_echo_experts ............................ None
326
+ moe_pad_expert_input_to_capacity ................ False
327
+ moe_per_layer_logging ........................... False
328
+ moe_permute_fusion .............................. False
329
+ moe_received_token_capacity ..................... None
330
+ moe_router_bias_update_rate ..................... 0.001
331
+ moe_router_dtype ................................ fp32
332
+ moe_router_enable_expert_bias ................... False
333
+ moe_router_force_load_balancing ................. False
334
+ moe_router_group_topk ........................... None
335
+ moe_router_load_balancing_type .................. aux_loss
336
+ moe_router_num_groups ........................... None
337
+ moe_router_padding_for_fp8 ...................... False
338
+ moe_router_pre_softmax .......................... False
339
+ moe_router_score_function ....................... softmax
340
+ moe_router_topk ................................. 32
341
+ moe_router_topk_scaling_factor .................. None
342
+ moe_shared_expert_intermediate_size ............. None
343
+ moe_shared_expert_overlap ....................... False
344
+ moe_token_dispatcher_type ....................... flex
345
+ moe_token_drop_policy ........................... probs
346
+ moe_upcycling_granularity ....................... 1
347
+ moe_use_device_initiated_grouped_gemm ........... True
348
+ moe_use_legacy_grouped_gemm ..................... False
349
+ moe_use_upcycling ............................... False
350
+ moe_z_loss_coeff ................................ None
351
+ mrope_section ................................... None
352
+ mscale .......................................... 1.0
353
+ mscale_all_dim .................................. 0.0
354
+ mtp_loss_scaling_factor ......................... 0.1
355
+ mtp_num_layers .................................. None
356
+ multi_latent_attention .......................... False
357
+ nccl_all_reduce_for_prefill ..................... False
358
+ nccl_communicator_config_path ................... None
359
+ nccl_ub ......................................... False
360
+ no_load_optim ................................... None
361
+ no_load_rng ..................................... None
362
+ no_persist_layer_norm ........................... False
363
+ no_rope_freq .................................... None
364
+ no_save_optim ................................... None
365
+ no_save_rng ..................................... None
366
+ non_persistent_ckpt_type ........................ None
367
+ non_persistent_global_ckpt_dir .................. None
368
+ non_persistent_local_ckpt_algo .................. fully_parallel
369
+ non_persistent_local_ckpt_dir ................... None
370
+ non_persistent_save_interval .................... None
371
+ norm_epsilon .................................... 1e-05
372
+ normalization ................................... RMSNorm
373
+ num_attention_heads ............................. 32
374
+ num_channels .................................... 3
375
+ num_classes ..................................... 1000
376
+ num_dataset_builder_threads ..................... 1
377
+ num_distributed_optimizer_instances ............. 1
378
+ num_experts ..................................... 128
379
+ num_layers ...................................... 16
380
+ num_layers_at_end_in_bf16 ....................... 1
381
+ num_layers_at_start_in_bf16 ..................... 1
382
+ num_layers_per_virtual_pipeline_stage ........... None
383
+ num_query_groups ................................ 8
384
+ num_virtual_stages_per_pipeline_rank ............ None
385
+ num_workers ..................................... 2
386
+ object_storage_cache_path ....................... None
387
+ one_logger_async ................................ False
388
+ one_logger_project .............................. megatron-lm
389
+ one_logger_run_name ............................. None
390
+ onnx_safe ....................................... None
391
+ openai_gelu ..................................... False
392
+ optimizer ....................................... adam
393
+ optimizer_cpu_offload ........................... False
394
+ optimizer_offload_fraction ...................... 1.0
395
+ output_bert_embeddings .......................... False
396
+ overlap_cpu_optimizer_d2h_h2d ................... False
397
+ overlap_grad_reduce ............................. True
398
+ overlap_moe_expert_parallel_comm ................ False
399
+ overlap_p2p_comm ................................ False
400
+ overlap_p2p_comm_warmup_flush ................... False
401
+ overlap_param_gather ............................ True
402
+ overlap_param_gather_with_optimizer_step ........ False
403
+ override_opt_param_scheduler .................... False
404
+ params_dtype .................................... torch.bfloat16
405
+ patch_dim ....................................... 16
406
+ per_split_data_args_path ........................ None
407
+ perform_initialization .......................... True
408
+ pin_cpu_grads ................................... True
409
+ pin_cpu_params .................................. True
410
+ pipeline_model_parallel_comm_backend ............ None
411
+ pipeline_model_parallel_layout .................. None
412
+ pipeline_model_parallel_size .................... 1
413
+ position_embedding_type ......................... rope
414
+ pretrained_checkpoint ........................... None
415
+ profile ......................................... True
416
+ profile_ranks ................................... [0, 1, 2, 3, 4, 5, 6, 7]
417
+ profile_step_end ................................ 8
418
+ profile_step_start .............................. 6
419
+ q_lora_rank ..................................... None
420
+ qk_head_dim ..................................... 128
421
+ qk_l2_norm ...................................... False
422
+ qk_layernorm .................................... False
423
+ qk_pos_emb_head_dim ............................. 64
424
+ query_in_block_prob ............................. 0.1
425
+ rampup_batch_size ............................... None
426
+ rank ............................................ 0
427
+ recompute_granularity ........................... None
428
+ recompute_method ................................ None
429
+ recompute_modules ............................... None
430
+ recompute_num_layers ............................ None
431
+ record_memory_history ........................... False
432
+ relative_attention_max_distance ................. 128
433
+ relative_attention_num_buckets .................. 32
434
+ replication ..................................... False
435
+ replication_factor .............................. 2
436
+ replication_jump ................................ None
437
+ rerun_mode ...................................... validate_results
438
+ reset_attention_mask ............................ False
439
+ reset_position_ids .............................. False
440
+ result_rejected_tracker_filename ................ None
441
+ retriever_report_topk_accuracies ................ []
442
+ retriever_score_scaling ......................... False
443
+ retriever_seq_length ............................ 256
444
+ retro_add_retriever ............................. False
445
+ retro_attention_gate ............................ 1
446
+ retro_cyclic_train_iters ........................ None
447
+ retro_encoder_attention_dropout ................. 0.1
448
+ retro_encoder_hidden_dropout .................... 0.1
449
+ retro_encoder_layers ............................ 2
450
+ retro_num_neighbors ............................. 2
451
+ retro_num_retrieved_chunks ...................... 2
452
+ retro_project_dir ............................... None
453
+ retro_verify_neighbor_count ..................... True
454
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
455
+ rope_scaling_factor ............................. 8.0
456
+ rope_type ....................................... None
457
+ rotary_base ..................................... 1000000
458
+ rotary_interleaved .............................. False
459
+ rotary_percent .................................. 1.0
460
+ rotary_scaling_factor ........................... 1.0
461
+ rotary_seq_len_interpolation_factor ............. None
462
+ run_workload_inspector_server ................... False
463
+ sample_rate ..................................... 1.0
464
+ save ............................................ None
465
+ save_interval ................................... None
466
+ save_retain_interval ............................ None
467
+ scatter_gather_tensors_in_pipeline .............. True
468
+ seed ............................................ 1234
469
+ seq_length ...................................... 4096
470
+ sequence_parallel ............................... False
471
+ sft ............................................. False
472
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
473
+ sgd_momentum .................................... 0.9
474
+ short_seq_prob .................................. 0.1
475
+ skip_train ...................................... False
476
+ skipped_train_samples ........................... 0
477
+ spec ............................................ None
478
+ split ........................................... 100,0,0
479
+ squared_relu .................................... False
480
+ start_weight_decay .............................. 0.1
481
+ straggler_ctrlr_port ............................ 65535
482
+ straggler_minmax_count .......................... 1
483
+ suggested_communication_unit_size ............... None
484
+ swiglu .......................................... True
485
+ swin_backbone_type .............................. tiny
486
+ symmetric_ar_type ............................... None
487
+ te_rng_tracker .................................. True
488
+ tensor_model_parallel_size ...................... 1
489
+ tensorboard_dir ................................. None
490
+ tensorboard_log_interval ........................ 1
491
+ tensorboard_queue_size .......................... 1000
492
+ test_data_path .................................. None
493
+ test_mode ....................................... False
494
+ tiktoken_num_special_tokens ..................... 1000
495
+ tiktoken_pattern ................................ None
496
+ tiktoken_special_tokens ......................... None
497
+ timing_log_level ................................ 0
498
+ timing_log_option ............................... minmax
499
+ titles_data_path ................................ None
500
+ tokenizer_model ................................. None
501
+ tokenizer_type .................................. GPT2BPETokenizer
502
+ torch_fsdp2_reshard_after_forward ............... True
503
+ tp_comm_bootstrap_backend ....................... nccl
504
+ tp_comm_bulk_dgrad .............................. True
505
+ tp_comm_bulk_wgrad .............................. True
506
+ tp_comm_overlap ................................. False
507
+ tp_comm_overlap_ag .............................. True
508
+ tp_comm_overlap_cfg ............................. None
509
+ tp_comm_overlap_rs .............................. True
510
+ tp_comm_overlap_rs_dgrad ........................ False
511
+ tp_comm_split_ag ................................ True
512
+ tp_comm_split_rs ................................ True
513
+ train_data_path ................................. None
514
+ train_iters ..................................... 10
515
+ train_samples ................................... None
516
+ train_sync_interval ............................. None
517
+ transformer_impl ................................ transformer_engine
518
+ transformer_pipeline_model_parallel_size ........ 1
519
+ untie_embeddings_and_output_weights ............. True
520
+ use_checkpoint_args ............................. False
521
+ use_checkpoint_opt_param_scheduler .............. False
522
+ use_cpu_initialization .......................... None
523
+ use_custom_fsdp ................................. False
524
+ use_dist_ckpt ................................... True
525
+ use_dist_ckpt_deprecated ........................ False
526
+ use_distributed_optimizer ....................... True
527
+ use_flash_attn .................................. False
528
+ use_legacy_models ............................... False
529
+ use_mp_args_from_checkpoint_args ................ False
530
+ use_one_sent_docs ............................... False
531
+ use_persistent_ckpt_worker ...................... False
532
+ use_precision_aware_optimizer ................... False
533
+ use_pytorch_profiler ............................ False
534
+ use_ring_exchange_p2p ........................... False
535
+ use_rope_scaling ................................ False
536
+ use_rotary_position_embeddings .................. False
537
+ use_sharp ....................................... False
538
+ use_tokenizer_model_from_checkpoint_args ........ True
539
+ use_torch_fsdp2 ................................. False
540
+ use_torch_optimizer_for_cpu_offload ............. False
541
+ use_tp_pp_dp_mapping ............................ False
542
+ v_head_dim ...................................... 128
543
+ valid_data_path ................................. None
544
+ variable_seq_lengths ............................ False
545
+ virtual_pipeline_model_parallel_size ............ None
546
+ vision_backbone_type ............................ vit
547
+ vision_pretraining .............................. False
548
+ vision_pretraining_type ......................... classify
549
+ vocab_extra_ids ................................. 0
550
+ vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json
551
+ vocab_size ...................................... None
552
+ wandb_exp_name ..................................
553
+ wandb_project ...................................
554
+ wandb_save_dir ..................................
555
+ weight_decay .................................... 0.1
556
+ weight_decay_incr_style ......................... constant
557
+ wgrad_deferral_limit ............................ 0
558
+ world_size ...................................... 8
559
+ yaml_cfg ........................................ None
560
+ -------------------- end of arguments ---------------------
561
+ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2
562
+ > building GPT2BPETokenizer tokenizer ...
563
+ > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304)
564
+ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS
565
+ > initializing torch distributed ...
566
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
567
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
568
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
569
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
570
+ WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it
571
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
572
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
573
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
574
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
575
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
576
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
577
+ /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled
578
+ warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled")
579
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
580
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
581
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
582
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
583
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
584
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
585
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
586
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
587
+ [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
588
+ [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
589
+ [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
590
+ [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
591
+ [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
592
+ [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
593
+ [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
594
+ [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7
595
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
596
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
597
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
598
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
599
+ > initialized tensor model parallel with size 1
600
+ [Gloo] Rank 0 is connected to > initialized pipeline model parallel with size 10 peer ranks.
601
+ Expected number of connected peer ranks is : 0
602
+ > setting random seeds to 1234 ...
603
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
604
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
605
+ [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
606
+ [rank6]:[W830 09:43:17.284436708 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
607
+ [rank2]:[W830 09:43:17.284458431 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
608
+ [rank1]:[W830 09:43:17.289279717 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
609
+ [rank4]:[W830 09:43:17.289305872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
610
+ [rank7]:[W830 09:43:17.289755909 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
611
+ [rank3]:[W830 09:43:17.290126880 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
612
+ [rank5]:[W830 09:43:17.290233819 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
613
+ > compiling dataset index builder ...
614
+ make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
615
+ make: Nothing to be done for 'default'.
616
+ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets'
617
+ >>> done with dataset index builder. Compilation time: 0.430 seconds
618
+ WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations.
619
+ > compiling and loading fused kernels ...
620
+ [rank0]:[W830 09:43:17.906751992 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device.
621
+ >>> done with compiling and loading fused kernels. Compilation time: 3.671 seconds
622
+ time to initialize megatron (seconds): 9.364
623
+ [after megatron is initialized] datetime: 2026-08-30 09:43:23
624
+ building GPT model ...
625
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
626
+ warnings.warn(
627
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
628
+ warnings.warn(
629
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
630
+ warnings.warn(
631
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
632
+ warnings.warn(
633
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
634
+ warnings.warn(
635
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
636
+ warnings.warn(
637
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
638
+ warnings.warn(
639
+ /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect.
640
+ warnings.warn(
641
+ > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3910275072
642
+ INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False)
643
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18
644
+ Params for bucket 1 (206045184 elements, 206045184 padded size):
645
+ module.output_layer.weight
646
+ Params for bucket 2 (42479616 elements, 42479616 padded size):
647
+ module.decoder.final_layernorm.weight
648
+ module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight
649
+ module.decoder.layers.15.mlp.router.weight
650
+ module.decoder.layers.15.self_attention.linear_qkv.weight
651
+ module.decoder.layers.15.pre_mlp_layernorm.weight
652
+ module.decoder.layers.15.self_attention.linear_proj.weight
653
+ Params for bucket 3 (42475520 elements, 42475520 padded size):
654
+ module.decoder.layers.14.mlp.router.weight
655
+ module.decoder.layers.14.pre_mlp_layernorm.weight
656
+ module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight
657
+ module.decoder.layers.14.self_attention.linear_qkv.weight
658
+ module.decoder.layers.14.self_attention.linear_proj.weight
659
+ Params for bucket 4 (42475520 elements, 42475520 padded size):
660
+ module.decoder.layers.13.mlp.router.weight
661
+ module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight
662
+ module.decoder.layers.13.pre_mlp_layernorm.weight
663
+ module.decoder.layers.13.self_attention.linear_qkv.weight
664
+ module.decoder.layers.13.self_attention.linear_proj.weight
665
+ Params for bucket 5 (42475520 elements, 42475520 padded size):
666
+ module.decoder.layers.12.mlp.router.weight
667
+ module.decoder.layers.12.pre_mlp_layernorm.weight
668
+ module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight
669
+ module.decoder.layers.12.self_attention.linear_proj.weight
670
+ module.decoder.layers.12.self_attention.linear_qkv.weight
671
+ Params for bucket 6 (42475520 elements, 42475520 padded size):
672
+ module.decoder.layers.11.pre_mlp_layernorm.weight
673
+ module.decoder.layers.11.mlp.router.weight
674
+ module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight
675
+ module.decoder.layers.11.self_attention.linear_qkv.weight
676
+ module.decoder.layers.11.self_attention.linear_proj.weight
677
+ Params for bucket 7 (42475520 elements, 42475520 padded size):
678
+ module.decoder.layers.10.mlp.router.weight
679
+ module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight
680
+ module.decoder.layers.10.self_attention.linear_qkv.weight
681
+ module.decoder.layers.10.pre_mlp_layernorm.weight
682
+ module.decoder.layers.10.self_attention.linear_proj.weight
683
+ Params for bucket 8 (42475520 elements, 42475520 padded size):
684
+ module.decoder.layers.9.mlp.router.weight
685
+ module.decoder.layers.9.pre_mlp_layernorm.weight
686
+ module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight
687
+ module.decoder.layers.9.self_attention.linear_proj.weight
688
+ module.decoder.layers.9.self_attention.linear_qkv.weight
689
+ Params for bucket 9 (42475520 elements, 42475520 padded size):
690
+ module.decoder.layers.8.mlp.router.weight
691
+ module.decoder.layers.8.self_attention.linear_qkv.weight
692
+ module.decoder.layers.8.pre_mlp_layernorm.weight
693
+ module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight
694
+ module.decoder.layers.8.self_attention.linear_proj.weight
695
+ Params for bucket 10 (42475520 elements, 42475520 padded size):
696
+ module.decoder.layers.7.mlp.router.weight
697
+ module.decoder.layers.7.pre_mlp_layernorm.weight
698
+ module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight
699
+ module.decoder.layers.7.self_attention.linear_proj.weight
700
+ module.decoder.layers.7.self_attention.linear_qkv.weight
701
+ Params for bucket 11 (42475520 elements, 42475520 padded size):
702
+ module.decoder.layers.6.mlp.router.weight
703
+ module.decoder.layers.6.self_attention.linear_qkv.weight
704
+ module.decoder.layers.6.self_attention.linear_proj.weight
705
+ module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight
706
+ module.decoder.layers.6.pre_mlp_layernorm.weight
707
+ Params for bucket 12 (42475520 elements, 42475520 padded size):
708
+ module.decoder.layers.5.mlp.router.weight
709
+ module.decoder.layers.5.pre_mlp_layernorm.weight
710
+ module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight
711
+ module.decoder.layers.5.self_attention.linear_proj.weight
712
+ module.decoder.layers.5.self_attention.linear_qkv.weight
713
+ Params for bucket 13 (42475520 elements, 42475520 padded size):
714
+ module.decoder.layers.4.mlp.router.weight
715
+ module.decoder.layers.4.self_attention.linear_qkv.weight
716
+ module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight
717
+ module.decoder.layers.4.self_attention.linear_proj.weight
718
+ module.decoder.layers.4.pre_mlp_layernorm.weight
719
+ Params for bucket 14 (42475520 elements, 42475520 padded size):
720
+ module.decoder.layers.3.mlp.router.weight
721
+ module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight
722
+ module.decoder.layers.3.self_attention.linear_qkv.weight
723
+ module.decoder.layers.3.self_attention.linear_proj.weight
724
+ module.decoder.layers.3.pre_mlp_layernorm.weight
725
+ Params for bucket 15 (42475520 elements, 42475520 padded size):
726
+ module.decoder.layers.2.mlp.router.weight
727
+ module.decoder.layers.2.pre_mlp_layernorm.weight
728
+ module.decoder.layers.2.self_attention.linear_qkv.weight
729
+ module.decoder.layers.2.self_attention.linear_proj.weight
730
+ module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight
731
+ Params for bucket 16 (42475520 elements, 42475520 padded size):
732
+ module.decoder.layers.1.mlp.router.weight
733
+ module.decoder.layers.1.self_attention.linear_proj.weight
734
+ module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight
735
+ module.decoder.layers.1.pre_mlp_layernorm.weight
736
+ module.decoder.layers.1.self_attention.linear_qkv.weight
737
+ Params for bucket 17 (42475520 elements, 42475520 padded size):
738
+ module.decoder.layers.0.pre_mlp_layernorm.weight
739
+ module.decoder.layers.0.self_attention.linear_proj.weight
740
+ module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight
741
+ module.decoder.layers.0.mlp.router.weight
742
+ module.decoder.layers.0.self_attention.linear_qkv.weight
743
+ Params for bucket 18 (206045184 elements, 206045184 padded size):
744
+ module.embedding.word_embeddings.weight
745
+ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67
746
+ Params for bucket 1 (40370176 elements, 40370176 padded size):
747
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight1
748
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight3
749
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight5
750
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight7
751
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight9
752
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight0
753
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight2
754
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight4
755
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight6
756
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight8
757
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight10
758
+ Params for bucket 2 (40370176 elements, 40370176 padded size):
759
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight11
760
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight15
761
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight13
762
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight0
763
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight2
764
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight14
765
+ module.decoder.layers.15.mlp.experts.linear_fc2.weight12
766
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight1
767
+ Params for bucket 3 (44040192 elements, 44040192 padded size):
768
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight4
769
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight6
770
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight8
771
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight3
772
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight5
773
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight7
774
+ Params for bucket 4 (44040192 elements, 44040192 padded size):
775
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight12
776
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight10
777
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight14
778
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight9
779
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight11
780
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight13
781
+ Params for bucket 5 (40370176 elements, 40370176 padded size):
782
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight2
783
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight0
784
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight4
785
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight6
786
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight8
787
+ module.decoder.layers.15.mlp.experts.linear_fc1.weight15
788
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight3
789
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight1
790
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight5
791
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight7
792
+ Params for bucket 6 (40370176 elements, 40370176 padded size):
793
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight12
794
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight10
795
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight14
796
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight1
797
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight9
798
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight13
799
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight11
800
+ module.decoder.layers.14.mlp.experts.linear_fc2.weight15
801
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight0
802
+ Params for bucket 7 (44040192 elements, 44040192 padded size):
803
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight5
804
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight3
805
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight7
806
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight2
807
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight6
808
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight4
809
+ Params for bucket 8 (44040192 elements, 44040192 padded size):
810
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight9
811
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight11
812
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight13
813
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight8
814
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight12
815
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight10
816
+ Params for bucket 9 (40370176 elements, 40370176 padded size):
817
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight15
818
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight1
819
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight3
820
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight5
821
+ module.decoder.layers.14.mlp.experts.linear_fc1.weight14
822
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight0
823
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight2
824
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight4
825
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight6
826
+ Params for bucket 10 (40370176 elements, 40370176 padded size):
827
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight7
828
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight9
829
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight11
830
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight13
831
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight15
832
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight0
833
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight10
834
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight8
835
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight12
836
+ module.decoder.layers.13.mlp.experts.linear_fc2.weight14
837
+ Params for bucket 11 (44040192 elements, 44040192 padded size):
838
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight2
839
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight4
840
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight6
841
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight1
842
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight3
843
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight5
844
+ Params for bucket 12 (44040192 elements, 44040192 padded size):
845
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight10
846
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight8
847
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight12
848
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight7
849
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight9
850
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight11
851
+ Params for bucket 13 (40370176 elements, 40370176 padded size):
852
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight0
853
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight14
854
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight2
855
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight4
856
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight13
857
+ module.decoder.layers.13.mlp.experts.linear_fc1.weight15
858
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight1
859
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight3
860
+ Params for bucket 14 (40370176 elements, 40370176 padded size):
861
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight6
862
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight8
863
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight10
864
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight12
865
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight14
866
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight5
867
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight9
868
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight7
869
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight11
870
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight13
871
+ module.decoder.layers.12.mlp.experts.linear_fc2.weight15
872
+ Params for bucket 15 (44040192 elements, 44040192 padded size):
873
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight3
874
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight1
875
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight5
876
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight0
877
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight4
878
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight2
879
+ Params for bucket 16 (44040192 elements, 44040192 padded size):
880
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight7
881
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight9
882
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight11
883
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight6
884
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight10
885
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight8
886
+ Params for bucket 17 (40370176 elements, 40370176 padded size):
887
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight13
888
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight15
889
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight1
890
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight12
891
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight0
892
+ module.decoder.layers.12.mlp.experts.linear_fc1.weight14
893
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight2
894
+ Params for bucket 18 (40370176 elements, 40370176 padded size):
895
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight3
896
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight7
897
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight5
898
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight9
899
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight11
900
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight13
901
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight6
902
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight4
903
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight8
904
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight10
905
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight12
906
+ Params for bucket 19 (44040192 elements, 44040192 padded size):
907
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight2
908
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight0
909
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight15
910
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight4
911
+ module.decoder.layers.11.mlp.experts.linear_fc2.weight14
912
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight1
913
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight3
914
+ Params for bucket 20 (44040192 elements, 44040192 padded size):
915
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight8
916
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight6
917
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight10
918
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight5
919
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight9
920
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight7
921
+ Params for bucket 21 (40370176 elements, 40370176 padded size):
922
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight12
923
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight14
924
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight11
925
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight13
926
+ module.decoder.layers.11.mlp.experts.linear_fc1.weight15
927
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight0
928
+ Params for bucket 22 (40370176 elements, 40370176 padded size):
929
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight1
930
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight5
931
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight3
932
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight7
933
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight9
934
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight11
935
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight4
936
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight2
937
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight6
938
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight8
939
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight10
940
+ Params for bucket 23 (44040192 elements, 44040192 padded size):
941
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight15
942
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight13
943
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight0
944
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight2
945
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight12
946
+ module.decoder.layers.10.mlp.experts.linear_fc2.weight14
947
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight1
948
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight3
949
+ Params for bucket 24 (44040192 elements, 44040192 padded size):
950
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight4
951
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight6
952
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight8
953
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight7
954
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight5
955
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight9
956
+ Params for bucket 25 (44040192 elements, 44040192 padded size):
957
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight10
958
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight12
959
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight14
960
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight13
961
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight11
962
+ module.decoder.layers.10.mlp.experts.linear_fc1.weight15
963
+ Params for bucket 26 (40370176 elements, 40370176 padded size):
964
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight0
965
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight4
966
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight2
967
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight6
968
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight8
969
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight10
970
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight1
971
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight3
972
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight5
973
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight7
974
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight9
975
+ Params for bucket 27 (40370176 elements, 40370176 padded size):
976
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight14
977
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight12
978
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight1
979
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight11
980
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight13
981
+ module.decoder.layers.9.mlp.experts.linear_fc2.weight15
982
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight0
983
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight2
984
+ Params for bucket 28 (44040192 elements, 44040192 padded size):
985
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight3
986
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight7
987
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight5
988
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight4
989
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight6
990
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight8
991
+ Params for bucket 29 (44040192 elements, 44040192 padded size):
992
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight9
993
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight13
994
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight11
995
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight10
996
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight12
997
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight14
998
+ Params for bucket 30 (40370176 elements, 40370176 padded size):
999
+ module.decoder.layers.9.mlp.experts.linear_fc1.weight15
1000
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight1
1001
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight3
1002
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight5
1003
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight7
1004
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight2
1005
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight0
1006
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight4
1007
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight6
1008
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight8
1009
+ Params for bucket 31 (40370176 elements, 40370176 padded size):
1010
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight9
1011
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight13
1012
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight11
1013
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight15
1014
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight0
1015
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight12
1016
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight10
1017
+ module.decoder.layers.8.mlp.experts.linear_fc2.weight14
1018
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight1
1019
+ Params for bucket 32 (44040192 elements, 44040192 padded size):
1020
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight2
1021
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight4
1022
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight6
1023
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight5
1024
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight3
1025
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight7
1026
+ Params for bucket 33 (44040192 elements, 44040192 padded size):
1027
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight8
1028
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight10
1029
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight12
1030
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight11
1031
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight9
1032
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight13
1033
+ Params for bucket 34 (40370176 elements, 40370176 padded size):
1034
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight14
1035
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight0
1036
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight2
1037
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight4
1038
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight6
1039
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight1
1040
+ module.decoder.layers.8.mlp.experts.linear_fc1.weight15
1041
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight3
1042
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight5
1043
+ Params for bucket 35 (40370176 elements, 40370176 padded size):
1044
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight10
1045
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight8
1046
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight12
1047
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight14
1048
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight7
1049
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight11
1050
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight9
1051
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight13
1052
+ module.decoder.layers.7.mlp.experts.linear_fc2.weight15
1053
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight0
1054
+ Params for bucket 36 (44040192 elements, 44040192 padded size):
1055
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight1
1056
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight5
1057
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight3
1058
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight2
1059
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight4
1060
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight6
1061
+ Params for bucket 37 (44040192 elements, 44040192 padded size):
1062
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight7
1063
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight11
1064
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight9
1065
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight8
1066
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight10
1067
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight12
1068
+ Params for bucket 38 (40370176 elements, 40370176 padded size):
1069
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight13
1070
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight1
1071
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight15
1072
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight3
1073
+ module.decoder.layers.7.mlp.experts.linear_fc1.weight14
1074
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight0
1075
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight2
1076
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight4
1077
+ Params for bucket 39 (40370176 elements, 40370176 padded size):
1078
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight5
1079
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight7
1080
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight9
1081
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight11
1082
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight13
1083
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight15
1084
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight8
1085
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight6
1086
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight10
1087
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight12
1088
+ module.decoder.layers.6.mlp.experts.linear_fc2.weight14
1089
+ Params for bucket 40 (44040192 elements, 44040192 padded size):
1090
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight0
1091
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight2
1092
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight4
1093
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight3
1094
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight1
1095
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight5
1096
+ Params for bucket 41 (44040192 elements, 44040192 padded size):
1097
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight6
1098
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight10
1099
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight8
1100
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight7
1101
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight9
1102
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight11
1103
+ Params for bucket 42 (40370176 elements, 40370176 padded size):
1104
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight12
1105
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight14
1106
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight0
1107
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight2
1108
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight15
1109
+ module.decoder.layers.6.mlp.experts.linear_fc1.weight13
1110
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight1
1111
+ Params for bucket 43 (40370176 elements, 40370176 padded size):
1112
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight6
1113
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight4
1114
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight8
1115
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight10
1116
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight12
1117
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight3
1118
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight5
1119
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight7
1120
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight9
1121
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight11
1122
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight13
1123
+ Params for bucket 44 (44040192 elements, 44040192 padded size):
1124
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight14
1125
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight1
1126
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight3
1127
+ module.decoder.layers.5.mlp.experts.linear_fc2.weight15
1128
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight0
1129
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight2
1130
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight4
1131
+ Params for bucket 45 (44040192 elements, 44040192 padded size):
1132
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight5
1133
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight9
1134
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight7
1135
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight8
1136
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight6
1137
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight10
1138
+ Params for bucket 46 (40370176 elements, 40370176 padded size):
1139
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight11
1140
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight13
1141
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight15
1142
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight12
1143
+ module.decoder.layers.5.mlp.experts.linear_fc1.weight14
1144
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight0
1145
+ Params for bucket 47 (40370176 elements, 40370176 padded size):
1146
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight1
1147
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight3
1148
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight5
1149
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight7
1150
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight9
1151
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight11
1152
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight2
1153
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight4
1154
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight6
1155
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight8
1156
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight10
1157
+ Params for bucket 48 (44040192 elements, 44040192 padded size):
1158
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight15
1159
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight0
1160
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight13
1161
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight2
1162
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight12
1163
+ module.decoder.layers.4.mlp.experts.linear_fc2.weight14
1164
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight1
1165
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight3
1166
+ Params for bucket 49 (44040192 elements, 44040192 padded size):
1167
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight4
1168
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight8
1169
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight6
1170
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight5
1171
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight7
1172
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight9
1173
+ Params for bucket 50 (44040192 elements, 44040192 padded size):
1174
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight10
1175
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight14
1176
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight12
1177
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight11
1178
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight13
1179
+ module.decoder.layers.4.mlp.experts.linear_fc1.weight15
1180
+ Params for bucket 51 (40370176 elements, 40370176 padded size):
1181
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight1
1182
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight3
1183
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight5
1184
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight7
1185
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight9
1186
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight0
1187
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight4
1188
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight2
1189
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight6
1190
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight8
1191
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight10
1192
+ Params for bucket 52 (40370176 elements, 40370176 padded size):
1193
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight11
1194
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight15
1195
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight13
1196
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight0
1197
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight2
1198
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight14
1199
+ module.decoder.layers.3.mlp.experts.linear_fc2.weight12
1200
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight1
1201
+ Params for bucket 53 (44040192 elements, 44040192 padded size):
1202
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight4
1203
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight6
1204
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight8
1205
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight3
1206
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight7
1207
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight5
1208
+ Params for bucket 54 (44040192 elements, 44040192 padded size):
1209
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight10
1210
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight12
1211
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight14
1212
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight9
1213
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight11
1214
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight13
1215
+ Params for bucket 55 (40370176 elements, 40370176 padded size):
1216
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight2
1217
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight0
1218
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight4
1219
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight6
1220
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight8
1221
+ module.decoder.layers.3.mlp.experts.linear_fc1.weight15
1222
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight3
1223
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight1
1224
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight5
1225
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight7
1226
+ Params for bucket 56 (40370176 elements, 40370176 padded size):
1227
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight12
1228
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight10
1229
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight14
1230
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight0
1231
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight9
1232
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight11
1233
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight13
1234
+ module.decoder.layers.2.mlp.experts.linear_fc2.weight15
1235
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight1
1236
+ Params for bucket 57 (44040192 elements, 44040192 padded size):
1237
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight2
1238
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight6
1239
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight4
1240
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight3
1241
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight5
1242
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight7
1243
+ Params for bucket 58 (44040192 elements, 44040192 padded size):
1244
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight8
1245
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight12
1246
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight10
1247
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight9
1248
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight11
1249
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight13
1250
+ Params for bucket 59 (40370176 elements, 40370176 padded size):
1251
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight14
1252
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight0
1253
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight2
1254
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight4
1255
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight6
1256
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight1
1257
+ module.decoder.layers.2.mlp.experts.linear_fc1.weight15
1258
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight3
1259
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight5
1260
+ Params for bucket 60 (40370176 elements, 40370176 padded size):
1261
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight10
1262
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight8
1263
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight12
1264
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight14
1265
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight7
1266
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight11
1267
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight9
1268
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight13
1269
+ module.decoder.layers.1.mlp.experts.linear_fc2.weight15
1270
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight0
1271
+ Params for bucket 61 (44040192 elements, 44040192 padded size):
1272
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight1
1273
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight3
1274
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight5
1275
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight4
1276
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight2
1277
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight6
1278
+ Params for bucket 62 (44040192 elements, 44040192 padded size):
1279
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight7
1280
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight11
1281
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight9
1282
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight10
1283
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight8
1284
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight12
1285
+ Params for bucket 63 (40370176 elements, 40370176 padded size):
1286
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight13
1287
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight0
1288
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight15
1289
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight2
1290
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight4
1291
+ module.decoder.layers.1.mlp.experts.linear_fc1.weight14
1292
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight1
1293
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight3
1294
+ Params for bucket 64 (40370176 elements, 40370176 padded size):
1295
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight6
1296
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight8
1297
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight10
1298
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight12
1299
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight14
1300
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight5
1301
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight7
1302
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight9
1303
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight11
1304
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight13
1305
+ module.decoder.layers.0.mlp.experts.linear_fc2.weight15
1306
+ Params for bucket 65 (44040192 elements, 44040192 padded size):
1307
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight3
1308
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight2
1309
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight5
1310
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight0
1311
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight4
1312
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight1
1313
+ Params for bucket 66 (44040192 elements, 44040192 padded size):
1314
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight7
1315
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight9
1316
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight11
1317
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight6
1318
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight10
1319
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight8
1320
+ Params for bucket 67 (29360128 elements, 29360128 padded size):
1321
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight12
1322
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight14
1323
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight15
1324
+ module.decoder.layers.0.mlp.experts.linear_fc1.weight13
1325
+ INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=<megatron.core.timers.Timers object at 0x7367ace09610>, config_logger_dir='')
1326
+ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine
1327
+ [after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 09:43:27
1328
+ > building train, validation, and test datasets ...
1329
+ > datasets target sizes (minimum size):
1330
+ train: 160
1331
+ validation: 0
1332
+ test: 0
1333
+ INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None]
1334
+ > building train, validation, and test datasets for GPT ...
1335
+ INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=<megatron.training.tokenizer.tokenizer._GPT2BPETokenizer object at 0x7367b7f099a0>, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None)
1336
+ INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx
1337
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths
1338
+ INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers
1339
+ INFO:megatron.core.datasets.indexed_dataset: Extract the document indices
1340
+ INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029
1341
+ INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350
1342
+ INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices
1343
+ INFO:megatron.core.datasets.gpt_dataset: Load the document index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-document_index.npy
1344
+ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-sample_index.npy
1345
+ INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy
1346
+ INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073
1347
+ > finished creating GPT datasets ...
1348
+ [after dataloaders are built] datetime: 2026-08-30 09:43:27
1349
+ done with setup ...
1350
+ training ...
1351
+ (min, max) time across ranks (ms):
1352
+ model-and-optimizer-setup ......................: (3356.97, 3386.96)
1353
+ train/valid/test-data-iterators-setup ..........: (85.46, 101.70)
1354
+ Overwriting rerun_state_machine.current_iteration from -1 to 0...
1355
+ [before the start of training step] datetime: 2026-08-30 09:43:27
1356
+ INFO:moonep.api:MoonEP comm buffer sizes:
1357
+ config : R=8, S=4096, H=4096, E=128, K=32, NvS=135136, NvS_padded=135168
1358
+ sm_count : num_sms=32, num_sms_dedup=148
1359
+ hidden_buffer_size : 1.03 GiB
1360
+ meta_buffer_size : 4.00 MiB
1361
+ local_temp_buffer_size : 4.29 MiB
1362
+ total_size : 1.04 GiB
1363
+ [MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [896, 22528, 5888, 2816, 10880, 3968, 7424, 7040, 8448, 1664, 9728, 2560, 7552, 5888, 1664, 3456, 23936, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[32, 40, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [103, 107, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [82, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1364
+ [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [6400, 256, 22528, 5376, 22912, 128, 1152, 22016, 0, 6656, 4480, 768, 15488, 4480, 128, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [98, 106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [91, 80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [8, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [100, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [102, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1365
+ [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [21632, 128, 26496, 5248, 7168, 24064, 128, 19328, 128, 0, 256, 0, 14592, 6656, 384, 3840, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, 85, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [9, 11, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1366
+ [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 1280, 16640, 6144, 7680, 1280, 3712, 2048, 22912, 18304, 10496, 0, 8960, 22528, 1664, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[98, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, 93, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [59, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [16, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]]
1367
+ [2026-08-30 09:43:49] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 21840.0 | throughput per GPU (TFLOP/s/GPU): 15.3 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102493E+01 | loss scale: 1.0 | grad norm: 44.120 | number of skipped iterations: 0 | number of nan iterations: 0 |
1368
+ Number of parameters in transformer block in billions: 23.22
1369
+ Number of parameters in embedding layers in billions: 0.41
1370
+ Total number of parameters in billions: 23.63
1371
+ Number of parameters in most loaded shard in billions: 23.6320
1372
+ Theoretical memory footprints: weight and optimizer=169029.43 MB
1373
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 53108.53564453125 | max allocated: 55305.33447265625 | reserved: 57154.0 | max reserved: 57154.0
1374
+ [2026-08-30 09:43:50] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 1284.3 | throughput per GPU (TFLOP/s/GPU): 259.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101859E+01 | loss scale: 1.0 | grad norm: 45.887 | number of skipped iterations: 0 | number of nan iterations: 0 |
1375
+ [2026-08-30 09:43:51] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 721.4 | throughput per GPU (TFLOP/s/GPU): 462.1 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.060162E+01 | loss scale: 1.0 | grad norm: 216.508 | number of skipped iterations: 0 | number of nan iterations: 0 |
1376
+ Capture CUDA graph for training!!!
1377
+ CUDA graph capture done!!!
1378
+ [2026-08-30 09:43:53] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 1792.2 | throughput per GPU (TFLOP/s/GPU): 186.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.301611E+01 | loss scale: 1.0 | grad norm: 46.056 | number of skipped iterations: 0 | number of nan iterations: 0 |
1379
+ [2026-08-30 09:43:53] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 785.1 | throughput per GPU (TFLOP/s/GPU): 424.6 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.033488E+01 | loss scale: 1.0 | grad norm: 17.387 | number of skipped iterations: 0 | number of nan iterations: 0 |
1380
+ [2026-08-30 09:43:54] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 787.7 | throughput per GPU (TFLOP/s/GPU): 423.2 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016456E+01 | loss scale: 1.0 | grad norm: 17.580 | number of skipped iterations: 0 | number of nan iterations: 0 |
1381
+ [2026-08-30 09:43:58] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3592.9 | throughput per GPU (TFLOP/s/GPU): 92.8 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.899017E+00 | loss scale: 1.0 | grad norm: 6.187 | number of skipped iterations: 0 | number of nan iterations: 0 |
1382
+ [2026-08-30 09:43:59] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 824.7 | throughput per GPU (TFLOP/s/GPU): 404.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.319439E+00 | loss scale: 1.0 | grad norm: 5.515 | number of skipped iterations: 0 | number of nan iterations: 0 |
tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv CHANGED
@@ -1 +1,4 @@
1
  model backend capacity status report async_overlap_json
 
 
 
 
1
  model backend capacity status report async_overlap_json
2
+ G1 moonep none OK /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
3
+ G2 moonep none OK /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
4
+ G3 moonep none OK /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
tiera-rerun-fixed-moonep/study.log CHANGED
@@ -2057,3 +2057,19 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main
2057
  [09:32:42] marker extension ready: _moonep_trace_C.cpython-312-x86_64-linux-gnu.so
2058
  [09:32:42] TRACE-ONLY G1__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7)
2059
  [guard] iter 6/10 (60s)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2057
  [09:32:42] marker extension ready: _moonep_trace_C.cpython-312-x86_64-linux-gnu.so
2058
  [09:32:42] TRACE-ONLY G1__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7)
2059
  [guard] iter 6/10 (60s)
2060
+ [guard] iter 8/10 (70s)
2061
+ [guard] iter 10/10 (110s)
2062
+ [guard] nsys stop (flushing report)
2063
+ [09:35:04] trace verified -> G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep (9.3M, all 8 GPUs + moonep markers)
2064
+ [09:35:05] TRACE-ONLY G2__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7)
2065
+ [guard] iter 6/10 (60s)
2066
+ [guard] iter 8/10 (70s)
2067
+ [09:38:56] WARN: nsys run failed for G2__moonep__matched__seq4096__gbs16__capnone__async (checking report anyway)
2068
+ [09:38:57] trace verified -> G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep (10M, all 8 GPUs + moonep markers)
2069
+ [09:38:58] TRACE-ONLY G3__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7)
2070
+ [guard] iter 6/10 (60s)
2071
+ [guard] iter 8/10 (70s)
2072
+ [09:42:49] WARN: nsys run failed for G3__moonep__matched__seq4096__gbs16__capnone__async (checking report anyway)
2073
+ [09:42:50] trace verified -> G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep (12M, all 8 GPUs + moonep markers)
2074
+ [09:42:51] TRACE-ONLY G4__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7)
2075
+ [guard] iter 3/10 (60s)