AbhishekGhosh1998 commited on
Commit
785e12b
·
verified ·
1 Parent(s): 5a7959d

granularity results granularity_next_b200/tiera_topk_trace/moonep/tierbtrace @ 2026-09-21T06:19:34Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0

Browse files
Files changed (22) hide show
  1. .gitattributes +3 -0
  2. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
  3. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
  4. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
  5. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
  6. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +105 -0
  7. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
  8. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
  9. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
  10. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
  11. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
  12. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
  13. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +105 -0
  14. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
  15. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
  16. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
  17. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
  18. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
  19. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
  20. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
  21. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
  22. granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv +3 -0
.gitattributes CHANGED
@@ -106,3 +106,6 @@ granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G3_K8_
106
  granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
107
  granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
108
  granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
 
 
106
  granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
107
  granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
108
  granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
109
+ granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
110
+ granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
111
+ granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9bf77de49b53024ab595177d714037b276d691b7da6944ed42ddf794f157259f
3
+ size 13358738
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 9.808375,
4
+ "overlapped_ms_per_iteration": 9.171625,
5
+ "exposed_ms_per_iteration": 0.6365000000000001,
6
+ "overlap_pct": 93.50809894605376,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 124.743 ms elementwise_kernel
4
+ 87.460 ms device_kernel
5
+ 59.517 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
6
+ 48.986 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
7
+ 48.137 ms multi_tensor_apply_kernel
8
+ 47.454 ms cast_mxfp8_2D_kernel
9
+ 42.268 ms vectorized_elementwise_kernel
10
+ 25.040 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
11
+ 23.857 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
12
+ 16.600 ms unrolled_elementwise_kernel
13
+ REDUCE 14.560 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
14
+ 14.287 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
15
+ 10.740 ms ncclDevKernel_AllGather_RING_LL
16
+ 9.253 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
17
+ 8.507 ms triton_poi_fused__to_copy_cat_0
18
+ 8.264 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
19
+ 8.238 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
20
+ 7.766 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
21
+ REDUCE 6.528 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
22
+ 5.946 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
23
+ 5.532 ms setGroupedGemmWgradArguments
24
+ 5.057 ms gatherTopK
25
+ 4.984 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
26
+ 4.804 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
27
+ 4.532 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
28
+ 4.223 ms reduce_kernel
29
+ 4.047 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
30
+ 3.901 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
31
+ 3.854 ms rmsnorm_bwd_tuned_kernel
32
+ 3.782 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
33
+ 3.663 ms multi_tensor_swizzle_row_scaling_kernel
34
+ 3.564 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
35
+ 3.450 ms CatArrayBatchedCopy_alignedK_contig
36
+ 3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
37
+ 3.370 ms _scatter_gather_elementwise_kernel
38
+ 3.297 ms triton_red_fused__to_copy_mul_silu_sum_1
39
+ 3.055 ms multi_tensor_swizzle_col_scaling_kernel
40
+ 2.631 ms fused_rope_backward_kernel
41
+ 2.558 ms fused_rope_forward_kernel
42
+ 2.434 ms triton_poi_fused__to_copy_mul_silu_0
43
+
44
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 128 (64/iter)
46
+ reduce_grad sum : 10.544 ms/iter
47
+ reduce_grad busy (union) : 10.544 ms/iter
48
+ ... overlapped with compute: 9.888 ms/iter ( 93.8 %)
49
+ ... exposed on critical path: 0.656 ms/iter ( 6.2 %)
50
+ other GPU work (union) : 331.413 ms/iter
51
+ traced wall : 391.824 ms/iter
52
+ --> time hidden by overlap : 9.888 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 124.677 ms elementwise_kernel
56
+ 86.754 ms device_kernel
57
+ 59.448 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
58
+ 49.352 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
59
+ 48.224 ms multi_tensor_apply_kernel
60
+ 47.782 ms cast_mxfp8_2D_kernel
61
+ 42.326 ms vectorized_elementwise_kernel
62
+ 25.141 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
63
+ 23.750 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
64
+ 16.683 ms unrolled_elementwise_kernel
65
+ REDUCE 14.215 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
66
+ 13.948 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
67
+ 12.621 ms ncclDevKernel_AllGather_RING_LL
68
+ 9.583 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
69
+ 9.433 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
70
+ 8.489 ms triton_poi_fused__to_copy_cat_0
71
+ 8.208 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
72
+ 8.143 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
73
+ 7.661 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
74
+ REDUCE 6.497 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
75
+ 6.372 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
76
+ 5.575 ms setGroupedGemmWgradArguments
77
+ 5.163 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
78
+ 5.064 ms gatherTopK
79
+ 4.714 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
80
+ 4.477 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
81
+ 4.228 ms reduce_kernel
82
+ 4.125 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
83
+ 3.850 ms rmsnorm_bwd_tuned_kernel
84
+ 3.782 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
85
+ 3.761 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
86
+ 3.704 ms multi_tensor_swizzle_row_scaling_kernel
87
+ 3.517 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
88
+ 3.474 ms CatArrayBatchedCopy_alignedK_contig
89
+ 3.385 ms _scatter_gather_elementwise_kernel
90
+ 3.338 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
91
+ 3.303 ms triton_red_fused__to_copy_mul_silu_sum_1
92
+ 3.028 ms multi_tensor_swizzle_col_scaling_kernel
93
+ 2.632 ms fused_rope_backward_kernel
94
+ 2.538 ms fused_rope_forward_kernel
95
+
96
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 128 (64/iter)
98
+ reduce_grad sum : 10.356 ms/iter
99
+ reduce_grad busy (union) : 10.356 ms/iter
100
+ ... overlapped with compute: 9.710 ms/iter ( 93.8 %)
101
+ ... exposed on critical path: 0.646 ms/iter ( 6.2 %)
102
+ other GPU work (union) : 337.144 ms/iter
103
+ traced wall : 392.464 ms/iter
104
+ --> time hidden by overlap : 9.710 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 124.523 ms elementwise_kernel
108
+ 88.064 ms device_kernel
109
+ 59.370 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
110
+ 53.586 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
111
+ 48.233 ms multi_tensor_apply_kernel
112
+ 47.355 ms cast_mxfp8_2D_kernel
113
+ 42.019 ms vectorized_elementwise_kernel
114
+ 24.373 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
115
+ 23.815 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
116
+ 16.659 ms unrolled_elementwise_kernel
117
+ REDUCE 12.377 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
118
+ 10.193 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
119
+ 10.056 ms ncclDevKernel_AllGather_RING_LL
120
+ 8.815 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
121
+ 8.506 ms triton_poi_fused__to_copy_cat_0
122
+ 8.249 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
123
+ 8.006 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
124
+ 7.905 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
125
+ 7.707 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
126
+ 6.728 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
127
+ REDUCE 6.381 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
128
+ 5.551 ms setGroupedGemmWgradArguments
129
+ 5.062 ms gatherTopK
130
+ 4.755 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
131
+ 4.743 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
132
+ 4.515 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
133
+ 4.186 ms reduce_kernel
134
+ 4.086 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
135
+ 3.895 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
136
+ 3.795 ms rmsnorm_bwd_tuned_kernel
137
+ 3.790 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
138
+ 3.694 ms multi_tensor_swizzle_row_scaling_kernel
139
+ 3.687 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
140
+ 3.465 ms CatArrayBatchedCopy_alignedK_contig
141
+ 3.396 ms _scatter_gather_elementwise_kernel
142
+ 3.385 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
143
+ 3.298 ms triton_red_fused__to_copy_mul_silu_sum_1
144
+ 3.054 ms multi_tensor_swizzle_col_scaling_kernel
145
+ 2.633 ms fused_rope_backward_kernel
146
+ 2.468 ms fused_rope_forward_kernel
147
+
148
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 128 (64/iter)
150
+ reduce_grad sum : 9.379 ms/iter
151
+ reduce_grad busy (union) : 9.379 ms/iter
152
+ ... overlapped with compute: 8.751 ms/iter ( 93.3 %)
153
+ ... exposed on critical path: 0.627 ms/iter ( 6.7 %)
154
+ other GPU work (union) : 335.373 ms/iter
155
+ traced wall : 392.194 ms/iter
156
+ --> time hidden by overlap : 8.751 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 124.615 ms elementwise_kernel
160
+ 88.174 ms device_kernel
161
+ 59.359 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
162
+ 54.730 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
163
+ 48.247 ms multi_tensor_apply_kernel
164
+ 47.377 ms cast_mxfp8_2D_kernel
165
+ 42.251 ms vectorized_elementwise_kernel
166
+ 24.065 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
167
+ 23.780 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
168
+ 16.663 ms unrolled_elementwise_kernel
169
+ REDUCE 11.624 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
170
+ 9.701 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
171
+ 8.513 ms triton_poi_fused__to_copy_cat_0
172
+ 8.506 ms ncclDevKernel_AllGather_RING_LL
173
+ 8.478 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
174
+ 8.313 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
175
+ 7.941 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
176
+ 7.807 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
177
+ 7.328 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
178
+ REDUCE 6.537 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
179
+ 6.219 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
180
+ 5.513 ms setGroupedGemmWgradArguments
181
+ 5.073 ms gatherTopK
182
+ 4.791 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
183
+ 4.601 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
184
+ 4.532 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
185
+ 4.188 ms reduce_kernel
186
+ 4.101 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
187
+ 3.971 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
188
+ 3.820 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
189
+ 3.729 ms rmsnorm_bwd_tuned_kernel
190
+ 3.652 ms multi_tensor_swizzle_row_scaling_kernel
191
+ 3.462 ms CatArrayBatchedCopy_alignedK_contig
192
+ 3.426 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
193
+ 3.406 ms _scatter_gather_elementwise_kernel
194
+ 3.297 ms triton_red_fused__to_copy_mul_silu_sum_1
195
+ 3.233 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
196
+ 3.058 ms multi_tensor_swizzle_col_scaling_kernel
197
+ 2.636 ms fused_rope_backward_kernel
198
+ 2.464 ms fused_rope_forward_kernel
199
+
200
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 128 (64/iter)
202
+ reduce_grad sum : 9.081 ms/iter
203
+ reduce_grad busy (union) : 9.081 ms/iter
204
+ ... overlapped with compute: 8.463 ms/iter ( 93.2 %)
205
+ ... exposed on critical path: 0.618 ms/iter ( 6.8 %)
206
+ other GPU work (union) : 334.445 ms/iter
207
+ traced wall : 391.299 ms/iter
208
+ --> time hidden by overlap : 8.463 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 124.595 ms elementwise_kernel
212
+ 87.627 ms device_kernel
213
+ 59.359 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
214
+ 54.267 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
215
+ 48.141 ms multi_tensor_apply_kernel
216
+ 47.381 ms cast_mxfp8_2D_kernel
217
+ 41.984 ms vectorized_elementwise_kernel
218
+ 24.408 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
219
+ 23.866 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
220
+ 16.683 ms unrolled_elementwise_kernel
221
+ REDUCE 12.149 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
222
+ 12.049 ms ncclDevKernel_AllGather_RING_LL
223
+ 10.317 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
224
+ 8.499 ms triton_poi_fused__to_copy_cat_0
225
+ 8.407 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
226
+ 8.276 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
227
+ 7.996 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
228
+ 7.764 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
229
+ 6.676 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
230
+ REDUCE 6.459 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
231
+ 5.776 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
232
+ 5.526 ms setGroupedGemmWgradArguments
233
+ 5.078 ms gatherTopK
234
+ 4.780 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
235
+ 4.529 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
236
+ 4.519 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
237
+ 4.197 ms reduce_kernel
238
+ 4.118 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
239
+ 3.937 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
240
+ 3.843 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
241
+ 3.806 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
242
+ 3.756 ms rmsnorm_bwd_tuned_kernel
243
+ 3.667 ms multi_tensor_swizzle_row_scaling_kernel
244
+ 3.469 ms CatArrayBatchedCopy_alignedK_contig
245
+ 3.377 ms _scatter_gather_elementwise_kernel
246
+ 3.367 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
247
+ 3.302 ms triton_red_fused__to_copy_mul_silu_sum_1
248
+ 3.023 ms multi_tensor_swizzle_col_scaling_kernel
249
+ 2.626 ms fused_rope_backward_kernel
250
+ 2.466 ms fused_rope_forward_kernel
251
+
252
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 128 (64/iter)
254
+ reduce_grad sum : 9.304 ms/iter
255
+ reduce_grad busy (union) : 9.304 ms/iter
256
+ ... overlapped with compute: 8.669 ms/iter ( 93.2 %)
257
+ ... exposed on critical path: 0.635 ms/iter ( 6.8 %)
258
+ other GPU work (union) : 335.415 ms/iter
259
+ traced wall : 392.463 ms/iter
260
+ --> time hidden by overlap : 8.669 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 124.633 ms elementwise_kernel
264
+ 87.722 ms device_kernel
265
+ 59.344 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
266
+ 53.161 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
267
+ 48.273 ms multi_tensor_apply_kernel
268
+ 47.349 ms cast_mxfp8_2D_kernel
269
+ 42.052 ms vectorized_elementwise_kernel
270
+ 24.002 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
271
+ 23.894 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
272
+ 16.733 ms unrolled_elementwise_kernel
273
+ 11.804 ms ncclDevKernel_AllGather_RING_LL
274
+ REDUCE 11.686 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
275
+ 11.387 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
276
+ 9.915 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
277
+ 9.385 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
278
+ 8.507 ms triton_poi_fused__to_copy_cat_0
279
+ 8.274 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
280
+ 7.948 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
281
+ 7.769 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
282
+ REDUCE 6.511 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
283
+ 5.519 ms setGroupedGemmWgradArguments
284
+ 5.378 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
285
+ 5.180 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
286
+ 5.067 ms gatherTopK
287
+ 4.770 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
288
+ 4.521 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
289
+ 4.216 ms reduce_kernel
290
+ 4.095 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
291
+ 3.893 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
292
+ 3.781 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
293
+ 3.747 ms rmsnorm_bwd_tuned_kernel
294
+ 3.595 ms multi_tensor_swizzle_row_scaling_kernel
295
+ 3.467 ms CatArrayBatchedCopy_alignedK_contig
296
+ 3.399 ms _scatter_gather_elementwise_kernel
297
+ 3.394 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
298
+ 3.302 ms triton_red_fused__to_copy_mul_silu_sum_1
299
+ 3.051 ms multi_tensor_swizzle_col_scaling_kernel
300
+ 2.935 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
301
+ 2.635 ms fused_rope_backward_kernel
302
+ 2.460 ms fused_rope_forward_kernel
303
+
304
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 128 (64/iter)
306
+ reduce_grad sum : 9.099 ms/iter
307
+ reduce_grad busy (union) : 9.099 ms/iter
308
+ ... overlapped with compute: 8.478 ms/iter ( 93.2 %)
309
+ ... exposed on critical path: 0.621 ms/iter ( 6.8 %)
310
+ other GPU work (union) : 337.042 ms/iter
311
+ traced wall : 392.314 ms/iter
312
+ --> time hidden by overlap : 8.478 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 124.699 ms elementwise_kernel
316
+ 87.979 ms device_kernel
317
+ 59.423 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
318
+ 50.071 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
319
+ 48.190 ms multi_tensor_apply_kernel
320
+ 47.864 ms cast_mxfp8_2D_kernel
321
+ 42.256 ms vectorized_elementwise_kernel
322
+ 24.570 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
323
+ 24.074 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
324
+ 16.660 ms unrolled_elementwise_kernel
325
+ REDUCE 14.139 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
326
+ 11.896 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
327
+ 11.593 ms ncclDevKernel_AllGather_RING_LL
328
+ 9.074 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
329
+ 8.843 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
330
+ 8.510 ms triton_poi_fused__to_copy_cat_0
331
+ 8.312 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
332
+ 8.236 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
333
+ 7.832 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
334
+ REDUCE 6.448 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
335
+ 6.282 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
336
+ 5.603 ms setGroupedGemmWgradArguments
337
+ 5.070 ms gatherTopK
338
+ 4.916 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
339
+ 4.787 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
340
+ 4.544 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
341
+ 4.215 ms reduce_kernel
342
+ 4.128 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
343
+ 4.009 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
344
+ 3.914 ms rmsnorm_bwd_tuned_kernel
345
+ 3.822 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
346
+ 3.680 ms multi_tensor_swizzle_row_scaling_kernel
347
+ 3.584 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
348
+ 3.465 ms CatArrayBatchedCopy_alignedK_contig
349
+ 3.411 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
350
+ 3.410 ms _scatter_gather_elementwise_kernel
351
+ 3.304 ms triton_red_fused__to_copy_mul_silu_sum_1
352
+ 3.053 ms multi_tensor_swizzle_col_scaling_kernel
353
+ 2.638 ms fused_rope_backward_kernel
354
+ 2.579 ms fused_rope_forward_kernel
355
+
356
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 128 (64/iter)
358
+ reduce_grad sum : 10.294 ms/iter
359
+ reduce_grad busy (union) : 10.294 ms/iter
360
+ ... overlapped with compute: 9.655 ms/iter ( 93.8 %)
361
+ ... exposed on critical path: 0.638 ms/iter ( 6.2 %)
362
+ other GPU work (union) : 336.380 ms/iter
363
+ traced wall : 391.953 ms/iter
364
+ --> time hidden by overlap : 9.655 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 124.623 ms elementwise_kernel
368
+ 86.503 ms device_kernel
369
+ 59.404 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
370
+ 49.040 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
371
+ 48.184 ms multi_tensor_apply_kernel
372
+ 47.912 ms cast_mxfp8_2D_kernel
373
+ 42.122 ms vectorized_elementwise_kernel
374
+ 25.215 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
375
+ 23.721 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
376
+ 16.655 ms unrolled_elementwise_kernel
377
+ 16.409 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
378
+ REDUCE 14.094 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
379
+ 10.327 ms ncclDevKernel_AllGather_RING_LL
380
+ 9.497 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
381
+ 8.490 ms triton_poi_fused__to_copy_cat_0
382
+ 8.216 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
383
+ 8.143 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
384
+ 7.796 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
385
+ 7.786 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
386
+ REDUCE 6.726 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
387
+ 5.624 ms setGroupedGemmWgradArguments
388
+ 5.126 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
389
+ 5.058 ms gatherTopK
390
+ 4.853 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
391
+ 4.769 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
392
+ 4.502 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
393
+ 4.216 ms reduce_kernel
394
+ 4.127 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
395
+ 3.869 ms rmsnorm_bwd_tuned_kernel
396
+ 3.838 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
397
+ 3.825 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
398
+ 3.666 ms multi_tensor_swizzle_row_scaling_kernel
399
+ 3.472 ms CatArrayBatchedCopy_alignedK_contig
400
+ 3.395 ms _scatter_gather_elementwise_kernel
401
+ 3.363 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
402
+ 3.303 ms triton_red_fused__to_copy_mul_silu_sum_1
403
+ 3.040 ms multi_tensor_swizzle_col_scaling_kernel
404
+ 3.002 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
405
+ 2.630 ms fused_rope_backward_kernel
406
+ 2.554 ms fused_rope_forward_kernel
407
+
408
+ === G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 128 (64/iter)
410
+ reduce_grad sum : 10.410 ms/iter
411
+ reduce_grad busy (union) : 10.410 ms/iter
412
+ ... overlapped with compute: 9.759 ms/iter ( 93.7 %)
413
+ ... exposed on critical path: 0.651 ms/iter ( 6.3 %)
414
+ other GPU work (union) : 335.222 ms/iter
415
+ traced wall : 391.638 ms/iter
416
+ --> time hidden by overlap : 9.759 ms/iter
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=9.808 ms/iter overlap=9.172 ms/iter (93.5%) exposed=0.637 ms/iter status=PASS
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=G4_K10
3
+ family=tierA
4
+ source_model=
5
+ standard_num_layers=
6
+ base_rung=G4
7
+ repo=/workspace/Megatron-MoE-Echo
8
+ branch=moonep_backend_async_grad_reduce
9
+ commit=2e68c8ad
10
+ moonep_source_dir=/workspace/MoonEP
11
+ moonep_source_branch=megatron-ddp-symmetric-params
12
+ moonep_source_commit=d8bc666
13
+ num_layers=16
14
+ hidden=4096
15
+ ffn_hidden=14336
16
+ moe_ffn_hidden=896
17
+ num_experts=128
18
+ topk=10
19
+ experts_per_rank=16
20
+ expert_params=11010048
21
+ heads=32
22
+ kv_groups=8
23
+ kv_channels=
24
+ qk_layernorm=0
25
+ shared_ffn=0
26
+ shared_expert_overlap=0
27
+ moe_token_dispatcher=flex
28
+ mla=0
29
+ q_lora_rank=
30
+ kv_lora_rank=512
31
+ qk_head_dim=128
32
+ qk_pos_emb_head_dim=64
33
+ v_head_dim=128
34
+ dense_first_layers=0
35
+ moe_layers=16
36
+ rotary_base=1000000
37
+ rotary_percent=1.0
38
+ rotary_scaling=
39
+ mscale=
40
+ mscale_all_dim=
41
+ qkv_bias=0
42
+ router_balancing=aux_loss
43
+ aux_loss_coeff=1e-2
44
+ router_pre_softmax=0
45
+ router_fusion=0
46
+ router_dtype=fp32
47
+ router_groups=
48
+ router_group_topk=
49
+ router_score_function=softmax
50
+ router_expert_bias=0
51
+ router_bias_update_rate=
52
+ router_topk_scaling_factor=
53
+ shared_expert_gate=0
54
+ apply_layernorm_1p=0
55
+ attention_output_gate=0
56
+ linear_attention_type=
57
+ linear_attention_freq=
58
+ linear_conv_kernel_dim=
59
+ linear_key_head_dim=
60
+ linear_value_head_dim=
61
+ linear_num_key_heads=
62
+ linear_num_value_heads=
63
+ mtp_num_layers=0
64
+ mtp_loss_scaling_factor=0.1
65
+ no_weight_decay_cond_type=
66
+ max_position_embeddings=32768
67
+ init_method_std=0.01
68
+ device_initiated_grouped_gemm=1
69
+ norm_epsilon=1e-5
70
+ seq_length=4096
71
+ micro_batch_size=1
72
+ global_batch_size=16
73
+ num_microbatches=2
74
+ assignments_per_rank=40960
75
+ mean_tokens_per_expert=2560
76
+ slot_policy=matched
77
+ slots_per_rank=16
78
+ capacity_mode=none
79
+ capacity=none
80
+ cuda_graph=1
81
+ probe_mode=0
82
+ async_grad_reduce=1
83
+ branch_role=timing
84
+ memory_dump_mode=none
85
+ memory_snapshot_path=
86
+ cuda_graph_warmup_steps=3
87
+ recompute=none
88
+ force_balance=0
89
+ train_iters=10
90
+ ep_size=8
91
+ echo_dispatch_overlap=1
92
+ echo_dispatch_recompute=0
93
+ ultraep_quota_min_tokens=
94
+ ultraep_weight_sync_join=
95
+ profile_enable=1
96
+ kernel_trace=1
97
+ fp8_mode=mxfp8
98
+ fp8_recipe=mxfp8
99
+ training_seed=1234
100
+ learning_rate=1e-4
101
+ min_learning_rate=1.0e-5
102
+ lr_warmup_iters=5
103
+ state_dump_enable=0
104
+ state_dump_dir=
105
+ moonep_assert_no_pending_reduce=1
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
2
+ kernels : 157894
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e2507c3838c92981cdfd7ecf187636d7cbc642492e01ecd6b8e154d381670cf8
3
+ size 13450239
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 9.724499999999999,
4
+ "overlapped_ms_per_iteration": 9.09,
5
+ "exposed_ms_per_iteration": 0.6345000000000001,
6
+ "overlap_pct": 93.47524294308191,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 185.953 ms elementwise_kernel
4
+ 130.258 ms device_kernel
5
+ 60.883 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
6
+ 58.596 ms cast_mxfp8_2D_kernel
7
+ 48.858 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
8
+ 48.675 ms vectorized_elementwise_kernel
9
+ 48.263 ms multi_tensor_apply_kernel
10
+ 24.913 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
11
+ 23.952 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
12
+ 23.884 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
13
+ 16.632 ms unrolled_elementwise_kernel
14
+ 13.739 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
15
+ 13.032 ms triton_poi_fused__to_copy_cat_0
16
+ REDUCE 12.732 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
17
+ 12.410 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
18
+ 11.694 ms ncclDevKernel_AllGather_RING_LL
19
+ 8.275 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
20
+ 8.078 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
21
+ 7.765 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
22
+ REDUCE 6.779 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
23
+ 5.716 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
24
+ 5.235 ms gatherTopK
25
+ 5.077 ms setGroupedGemmWgradArguments
26
+ 4.890 ms triton_red_fused__to_copy_mul_silu_sum_1
27
+ 4.796 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
28
+ 4.662 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
29
+ 4.546 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
30
+ 4.219 ms reduce_kernel
31
+ 4.191 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
32
+ 4.180 ms multi_tensor_swizzle_row_scaling_kernel
33
+ 3.911 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
34
+ 3.780 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
35
+ 3.768 ms rmsnorm_bwd_tuned_kernel
36
+ 3.696 ms _scatter_gather_elementwise_kernel
37
+ 3.672 ms triton_poi_fused__to_copy_mul_silu_0
38
+ 3.455 ms CatArrayBatchedCopy_alignedK_contig
39
+ 3.380 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
40
+ 3.175 ms multi_tensor_swizzle_col_scaling_kernel
41
+ 3.080 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
42
+ 2.628 ms fused_rope_backward_kernel
43
+
44
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 128 (64/iter)
46
+ reduce_grad sum : 9.755 ms/iter
47
+ reduce_grad busy (union) : 9.755 ms/iter
48
+ ... overlapped with compute: 9.130 ms/iter ( 93.6 %)
49
+ ... exposed on critical path: 0.625 ms/iter ( 6.4 %)
50
+ other GPU work (union) : 410.945 ms/iter
51
+ traced wall : 487.279 ms/iter
52
+ --> time hidden by overlap : 9.130 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 185.813 ms elementwise_kernel
56
+ 128.677 ms device_kernel
57
+ 60.727 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
58
+ 58.881 ms cast_mxfp8_2D_kernel
59
+ 48.818 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
60
+ 48.790 ms vectorized_elementwise_kernel
61
+ 48.201 ms multi_tensor_apply_kernel
62
+ 25.032 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
63
+ 24.216 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
64
+ 23.815 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
65
+ 16.747 ms unrolled_elementwise_kernel
66
+ 13.772 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
67
+ 13.492 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
68
+ 13.000 ms triton_poi_fused__to_copy_cat_0
69
+ REDUCE 12.626 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
70
+ 8.383 ms ncclDevKernel_AllGather_RING_LL
71
+ 8.209 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
72
+ 7.972 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
73
+ 7.645 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
74
+ REDUCE 6.845 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
75
+ 5.941 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
76
+ 5.221 ms gatherTopK
77
+ 5.036 ms setGroupedGemmWgradArguments
78
+ 4.900 ms triton_red_fused__to_copy_mul_silu_sum_1
79
+ 4.731 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
80
+ 4.473 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
81
+ 4.229 ms reduce_kernel
82
+ 4.217 ms multi_tensor_swizzle_row_scaling_kernel
83
+ 4.217 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
84
+ 3.778 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
85
+ 3.768 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
86
+ 3.740 ms _scatter_gather_elementwise_kernel
87
+ 3.705 ms rmsnorm_bwd_tuned_kernel
88
+ 3.661 ms triton_poi_fused__to_copy_mul_silu_0
89
+ 3.466 ms CatArrayBatchedCopy_alignedK_contig
90
+ 3.403 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
91
+ 3.391 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
92
+ 3.339 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
93
+ 3.126 ms multi_tensor_swizzle_col_scaling_kernel
94
+ 2.629 ms fused_rope_backward_kernel
95
+
96
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 128 (64/iter)
98
+ reduce_grad sum : 9.736 ms/iter
99
+ reduce_grad busy (union) : 9.736 ms/iter
100
+ ... overlapped with compute: 9.111 ms/iter ( 93.6 %)
101
+ ... exposed on critical path: 0.625 ms/iter ( 6.4 %)
102
+ other GPU work (union) : 408.592 ms/iter
103
+ traced wall : 487.269 ms/iter
104
+ --> time hidden by overlap : 9.111 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 185.818 ms elementwise_kernel
108
+ 130.847 ms device_kernel
109
+ 60.764 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
110
+ 58.518 ms cast_mxfp8_2D_kernel
111
+ 50.710 ms vectorized_elementwise_kernel
112
+ 48.898 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
113
+ 48.203 ms multi_tensor_apply_kernel
114
+ 25.222 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
115
+ 24.035 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
116
+ 23.886 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
117
+ 16.677 ms unrolled_elementwise_kernel
118
+ 13.868 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
119
+ 13.031 ms triton_poi_fused__to_copy_cat_0
120
+ REDUCE 12.681 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
121
+ 11.355 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
122
+ 8.933 ms ncclDevKernel_AllGather_RING_LL
123
+ 8.250 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
124
+ 8.120 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
125
+ 7.708 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
126
+ REDUCE 6.756 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
127
+ 6.103 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
128
+ 5.234 ms gatherTopK
129
+ 4.987 ms setGroupedGemmWgradArguments
130
+ 4.899 ms triton_red_fused__to_copy_mul_silu_sum_1
131
+ 4.878 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
132
+ 4.782 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
133
+ 4.533 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
134
+ 4.273 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
135
+ 4.194 ms reduce_kernel
136
+ 4.173 ms multi_tensor_swizzle_row_scaling_kernel
137
+ 3.905 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
138
+ 3.778 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
139
+ 3.729 ms rmsnorm_bwd_tuned_kernel
140
+ 3.725 ms _scatter_gather_elementwise_kernel
141
+ 3.669 ms triton_poi_fused__to_copy_mul_silu_0
142
+ 3.461 ms CatArrayBatchedCopy_alignedK_contig
143
+ 3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
144
+ 3.186 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
145
+ 3.162 ms multi_tensor_swizzle_col_scaling_kernel
146
+ 2.628 ms fused_rope_backward_kernel
147
+
148
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 128 (64/iter)
150
+ reduce_grad sum : 9.718 ms/iter
151
+ reduce_grad busy (union) : 9.718 ms/iter
152
+ ... overlapped with compute: 9.073 ms/iter ( 93.4 %)
153
+ ... exposed on critical path: 0.645 ms/iter ( 6.6 %)
154
+ other GPU work (union) : 410.582 ms/iter
155
+ traced wall : 530.417 ms/iter
156
+ --> time hidden by overlap : 9.073 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 185.985 ms elementwise_kernel
160
+ 131.612 ms device_kernel
161
+ 60.724 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
162
+ 58.618 ms cast_mxfp8_2D_kernel
163
+ 49.566 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
164
+ 48.779 ms vectorized_elementwise_kernel
165
+ 48.214 ms multi_tensor_apply_kernel
166
+ 24.563 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
167
+ 24.194 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
168
+ 23.389 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
169
+ 16.705 ms unrolled_elementwise_kernel
170
+ 13.570 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
171
+ 13.039 ms triton_poi_fused__to_copy_cat_0
172
+ REDUCE 12.524 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
173
+ 10.127 ms ncclDevKernel_AllGather_RING_LL
174
+ 9.674 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
175
+ 8.315 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
176
+ 8.114 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
177
+ 7.812 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
178
+ REDUCE 6.816 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
179
+ 5.911 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
180
+ 5.217 ms gatherTopK
181
+ 4.998 ms setGroupedGemmWgradArguments
182
+ 4.918 ms triton_red_fused__to_copy_mul_silu_sum_1
183
+ 4.808 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
184
+ 4.547 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
185
+ 4.287 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
186
+ 4.179 ms multi_tensor_swizzle_row_scaling_kernel
187
+ 4.177 ms reduce_kernel
188
+ 3.967 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
189
+ 3.827 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
190
+ 3.740 ms _scatter_gather_elementwise_kernel
191
+ 3.681 ms rmsnorm_bwd_tuned_kernel
192
+ 3.665 ms triton_poi_fused__to_copy_mul_silu_0
193
+ 3.470 ms CatArrayBatchedCopy_alignedK_contig
194
+ 3.421 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
195
+ 3.347 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
196
+ 3.163 ms multi_tensor_swizzle_col_scaling_kernel
197
+ 2.635 ms fused_rope_backward_kernel
198
+ 2.466 ms fused_rope_forward_kernel
199
+
200
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 128 (64/iter)
202
+ reduce_grad sum : 9.670 ms/iter
203
+ reduce_grad busy (union) : 9.670 ms/iter
204
+ ... overlapped with compute: 9.028 ms/iter ( 93.4 %)
205
+ ... exposed on critical path: 0.642 ms/iter ( 6.6 %)
206
+ other GPU work (union) : 408.736 ms/iter
207
+ traced wall : 487.194 ms/iter
208
+ --> time hidden by overlap : 9.028 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 185.905 ms elementwise_kernel
212
+ 129.675 ms device_kernel
213
+ 60.748 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
214
+ 58.671 ms cast_mxfp8_2D_kernel
215
+ 50.666 ms vectorized_elementwise_kernel
216
+ 48.255 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
217
+ 48.204 ms multi_tensor_apply_kernel
218
+ 25.777 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
219
+ 24.097 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
220
+ 23.903 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
221
+ 16.642 ms unrolled_elementwise_kernel
222
+ 14.072 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
223
+ 13.014 ms triton_poi_fused__to_copy_cat_0
224
+ REDUCE 12.736 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
225
+ 11.153 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
226
+ 9.216 ms ncclDevKernel_AllGather_RING_LL
227
+ 8.264 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
228
+ 8.074 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
229
+ 7.771 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
230
+ 6.940 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
231
+ REDUCE 6.753 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
232
+ 5.260 ms gatherTopK
233
+ 5.087 ms setGroupedGemmWgradArguments
234
+ 4.893 ms triton_red_fused__to_copy_mul_silu_sum_1
235
+ 4.809 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
236
+ 4.549 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
237
+ 4.274 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
238
+ 4.200 ms multi_tensor_swizzle_row_scaling_kernel
239
+ 4.195 ms reduce_kernel
240
+ 3.854 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
241
+ 3.792 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
242
+ 3.746 ms rmsnorm_bwd_tuned_kernel
243
+ 3.694 ms _scatter_gather_elementwise_kernel
244
+ 3.667 ms triton_poi_fused__to_copy_mul_silu_0
245
+ 3.470 ms CatArrayBatchedCopy_alignedK_contig
246
+ 3.364 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
247
+ 3.218 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
248
+ 3.155 ms multi_tensor_swizzle_col_scaling_kernel
249
+ 2.934 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
250
+ 2.632 ms fused_rope_backward_kernel
251
+
252
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 128 (64/iter)
254
+ reduce_grad sum : 9.745 ms/iter
255
+ reduce_grad busy (union) : 9.745 ms/iter
256
+ ... overlapped with compute: 9.104 ms/iter ( 93.4 %)
257
+ ... exposed on critical path: 0.641 ms/iter ( 6.6 %)
258
+ other GPU work (union) : 409.825 ms/iter
259
+ traced wall : 529.382 ms/iter
260
+ --> time hidden by overlap : 9.104 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 185.961 ms elementwise_kernel
264
+ 130.127 ms device_kernel
265
+ 60.748 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
266
+ 58.582 ms cast_mxfp8_2D_kernel
267
+ 48.736 ms vectorized_elementwise_kernel
268
+ 48.616 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
269
+ 48.259 ms multi_tensor_apply_kernel
270
+ 25.310 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
271
+ 23.960 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
272
+ 23.649 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
273
+ 16.814 ms unrolled_elementwise_kernel
274
+ 13.936 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
275
+ 13.029 ms triton_poi_fused__to_copy_cat_0
276
+ REDUCE 12.509 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
277
+ 12.045 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
278
+ 11.218 ms ncclDevKernel_AllGather_RING_LL
279
+ 8.262 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
280
+ 8.080 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
281
+ 7.759 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
282
+ REDUCE 6.862 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
283
+ 5.662 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
284
+ 5.464 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
285
+ 5.229 ms gatherTopK
286
+ 5.056 ms setGroupedGemmWgradArguments
287
+ 4.902 ms triton_red_fused__to_copy_mul_silu_sum_1
288
+ 4.776 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
289
+ 4.531 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
290
+ 4.266 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
291
+ 4.211 ms reduce_kernel
292
+ 4.152 ms multi_tensor_swizzle_row_scaling_kernel
293
+ 3.896 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
294
+ 3.786 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
295
+ 3.730 ms rmsnorm_bwd_tuned_kernel
296
+ 3.721 ms _scatter_gather_elementwise_kernel
297
+ 3.671 ms triton_poi_fused__to_copy_mul_silu_0
298
+ 3.464 ms CatArrayBatchedCopy_alignedK_contig
299
+ 3.391 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
300
+ 3.154 ms multi_tensor_swizzle_col_scaling_kernel
301
+ 2.983 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
302
+ 2.626 ms fused_rope_backward_kernel
303
+
304
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 128 (64/iter)
306
+ reduce_grad sum : 9.686 ms/iter
307
+ reduce_grad busy (union) : 9.686 ms/iter
308
+ ... overlapped with compute: 9.045 ms/iter ( 93.4 %)
309
+ ... exposed on critical path: 0.641 ms/iter ( 6.6 %)
310
+ other GPU work (union) : 410.918 ms/iter
311
+ traced wall : 487.069 ms/iter
312
+ --> time hidden by overlap : 9.045 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 185.903 ms elementwise_kernel
316
+ 130.934 ms device_kernel
317
+ 60.720 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
318
+ 59.068 ms cast_mxfp8_2D_kernel
319
+ 50.826 ms vectorized_elementwise_kernel
320
+ 49.246 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
321
+ 48.191 ms multi_tensor_apply_kernel
322
+ 24.750 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
323
+ 24.170 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
324
+ 23.382 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
325
+ 16.682 ms unrolled_elementwise_kernel
326
+ 13.654 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
327
+ 13.021 ms triton_poi_fused__to_copy_cat_0
328
+ REDUCE 12.749 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
329
+ 9.270 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
330
+ 8.888 ms ncclDevKernel_AllGather_RING_LL
331
+ 8.333 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
332
+ 8.148 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
333
+ 7.821 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
334
+ REDUCE 6.774 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
335
+ 6.714 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
336
+ 5.219 ms gatherTopK
337
+ 5.145 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
338
+ 5.099 ms setGroupedGemmWgradArguments
339
+ 4.911 ms triton_red_fused__to_copy_mul_silu_sum_1
340
+ 4.828 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
341
+ 4.575 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
342
+ 4.233 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
343
+ 4.219 ms reduce_kernel
344
+ 4.202 ms multi_tensor_swizzle_row_scaling_kernel
345
+ 4.021 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
346
+ 3.831 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
347
+ 3.774 ms rmsnorm_bwd_tuned_kernel
348
+ 3.722 ms _scatter_gather_elementwise_kernel
349
+ 3.659 ms triton_poi_fused__to_copy_mul_silu_0
350
+ 3.461 ms CatArrayBatchedCopy_alignedK_contig
351
+ 3.403 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
352
+ 3.338 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
353
+ 3.157 ms multi_tensor_swizzle_col_scaling_kernel
354
+ 2.633 ms fused_rope_backward_kernel
355
+
356
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 128 (64/iter)
358
+ reduce_grad sum : 9.761 ms/iter
359
+ reduce_grad busy (union) : 9.761 ms/iter
360
+ ... overlapped with compute: 9.129 ms/iter ( 93.5 %)
361
+ ... exposed on critical path: 0.632 ms/iter ( 6.5 %)
362
+ other GPU work (union) : 410.856 ms/iter
363
+ traced wall : 529.363 ms/iter
364
+ --> time hidden by overlap : 9.129 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 185.800 ms elementwise_kernel
368
+ 128.396 ms device_kernel
369
+ 60.718 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
370
+ 59.059 ms cast_mxfp8_2D_kernel
371
+ 50.852 ms vectorized_elementwise_kernel
372
+ 48.800 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
373
+ 48.163 ms multi_tensor_apply_kernel
374
+ 25.135 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
375
+ 23.793 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
376
+ 23.782 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
377
+ 16.650 ms unrolled_elementwise_kernel
378
+ 13.861 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
379
+ 13.304 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
380
+ 13.007 ms triton_poi_fused__to_copy_cat_0
381
+ REDUCE 12.573 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
382
+ 10.835 ms ncclDevKernel_AllGather_RING_LL
383
+ 8.228 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
384
+ 8.056 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
385
+ 7.804 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
386
+ REDUCE 6.878 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
387
+ 6.274 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
388
+ 5.954 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
389
+ 5.241 ms gatherTopK
390
+ 5.147 ms setGroupedGemmWgradArguments
391
+ 4.898 ms triton_red_fused__to_copy_mul_silu_sum_1
392
+ 4.798 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
393
+ 4.521 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
394
+ 4.230 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
395
+ 4.225 ms reduce_kernel
396
+ 4.217 ms multi_tensor_swizzle_row_scaling_kernel
397
+ 3.843 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
398
+ 3.823 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
399
+ 3.761 ms _scatter_gather_elementwise_kernel
400
+ 3.706 ms rmsnorm_bwd_tuned_kernel
401
+ 3.649 ms triton_poi_fused__to_copy_mul_silu_0
402
+ 3.464 ms CatArrayBatchedCopy_alignedK_contig
403
+ 3.424 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
404
+ 3.357 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
405
+ 3.141 ms multi_tensor_swizzle_col_scaling_kernel
406
+ 2.632 ms fused_rope_backward_kernel
407
+
408
+ === G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 128 (64/iter)
410
+ reduce_grad sum : 9.725 ms/iter
411
+ reduce_grad busy (union) : 9.725 ms/iter
412
+ ... overlapped with compute: 9.100 ms/iter ( 93.6 %)
413
+ ... exposed on critical path: 0.625 ms/iter ( 6.4 %)
414
+ other GPU work (union) : 412.420 ms/iter
415
+ traced wall : 530.283 ms/iter
416
+ --> time hidden by overlap : 9.100 ms/iter
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=9.724 ms/iter overlap=9.090 ms/iter (93.5%) exposed=0.635 ms/iter status=PASS
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ backend=moonep
2
+ config=G4_K16
3
+ family=tierA
4
+ source_model=
5
+ standard_num_layers=
6
+ base_rung=G4
7
+ repo=/workspace/Megatron-MoE-Echo
8
+ branch=moonep_backend_async_grad_reduce
9
+ commit=2e68c8ad
10
+ moonep_source_dir=/workspace/MoonEP
11
+ moonep_source_branch=megatron-ddp-symmetric-params
12
+ moonep_source_commit=d8bc666
13
+ num_layers=16
14
+ hidden=4096
15
+ ffn_hidden=14336
16
+ moe_ffn_hidden=896
17
+ num_experts=128
18
+ topk=16
19
+ experts_per_rank=16
20
+ expert_params=11010048
21
+ heads=32
22
+ kv_groups=8
23
+ kv_channels=
24
+ qk_layernorm=0
25
+ shared_ffn=0
26
+ shared_expert_overlap=0
27
+ moe_token_dispatcher=flex
28
+ mla=0
29
+ q_lora_rank=
30
+ kv_lora_rank=512
31
+ qk_head_dim=128
32
+ qk_pos_emb_head_dim=64
33
+ v_head_dim=128
34
+ dense_first_layers=0
35
+ moe_layers=16
36
+ rotary_base=1000000
37
+ rotary_percent=1.0
38
+ rotary_scaling=
39
+ mscale=
40
+ mscale_all_dim=
41
+ qkv_bias=0
42
+ router_balancing=aux_loss
43
+ aux_loss_coeff=1e-2
44
+ router_pre_softmax=0
45
+ router_fusion=0
46
+ router_dtype=fp32
47
+ router_groups=
48
+ router_group_topk=
49
+ router_score_function=softmax
50
+ router_expert_bias=0
51
+ router_bias_update_rate=
52
+ router_topk_scaling_factor=
53
+ shared_expert_gate=0
54
+ apply_layernorm_1p=0
55
+ attention_output_gate=0
56
+ linear_attention_type=
57
+ linear_attention_freq=
58
+ linear_conv_kernel_dim=
59
+ linear_key_head_dim=
60
+ linear_value_head_dim=
61
+ linear_num_key_heads=
62
+ linear_num_value_heads=
63
+ mtp_num_layers=0
64
+ mtp_loss_scaling_factor=0.1
65
+ no_weight_decay_cond_type=
66
+ max_position_embeddings=32768
67
+ init_method_std=0.01
68
+ device_initiated_grouped_gemm=1
69
+ norm_epsilon=1e-5
70
+ seq_length=4096
71
+ micro_batch_size=1
72
+ global_batch_size=16
73
+ num_microbatches=2
74
+ assignments_per_rank=65536
75
+ mean_tokens_per_expert=4096
76
+ slot_policy=matched
77
+ slots_per_rank=16
78
+ capacity_mode=none
79
+ capacity=none
80
+ cuda_graph=1
81
+ probe_mode=0
82
+ async_grad_reduce=1
83
+ branch_role=timing
84
+ memory_dump_mode=none
85
+ memory_snapshot_path=
86
+ cuda_graph_warmup_steps=3
87
+ recompute=none
88
+ force_balance=0
89
+ train_iters=10
90
+ ep_size=8
91
+ echo_dispatch_overlap=1
92
+ echo_dispatch_recompute=0
93
+ ultraep_quota_min_tokens=
94
+ ultraep_weight_sync_join=
95
+ profile_enable=1
96
+ kernel_trace=1
97
+ fp8_mode=mxfp8
98
+ fp8_recipe=mxfp8
99
+ training_seed=1234
100
+ learning_rate=1e-4
101
+ min_learning_rate=1.0e-5
102
+ lr_warmup_iters=5
103
+ state_dump_enable=0
104
+ state_dump_dir=
105
+ moonep_assert_no_pending_reduce=1
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log ADDED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
2
+ kernels : 157954
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc07abf7c635833117d47053bae06cc1a7ce369d15d022463debc970eca5bbff
3
+ size 13044490
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 9.074,
4
+ "overlapped_ms_per_iteration": 8.45875,
5
+ "exposed_ms_per_iteration": 0.61525,
6
+ "overlap_pct": 93.21963852766144,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 104.215 ms elementwise_kernel
4
+ 73.573 ms device_kernel
5
+ 60.108 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
6
+ 48.226 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
7
+ 48.201 ms multi_tensor_apply_kernel
8
+ 43.680 ms cast_mxfp8_2D_kernel
9
+ 39.886 ms vectorized_elementwise_kernel
10
+ 23.964 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
11
+ 23.859 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
12
+ 16.662 ms unrolled_elementwise_kernel
13
+ REDUCE 11.863 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
14
+ 11.483 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
15
+ 11.263 ms ncclDevKernel_AllGather_RING_LL
16
+ 8.275 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
17
+ 7.997 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
18
+ 7.762 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
19
+ 7.008 ms triton_poi_fused__to_copy_cat_0
20
+ REDUCE 6.568 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
21
+ 6.406 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
22
+ 5.652 ms setGroupedGemmWgradArguments
23
+ 4.999 ms gatherTopK
24
+ 4.796 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
25
+ 4.536 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
26
+ 4.216 ms reduce_kernel
27
+ 3.931 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
28
+ 3.900 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
29
+ 3.779 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
30
+ 3.679 ms rmsnorm_bwd_tuned_kernel
31
+ 3.525 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
32
+ 3.454 ms CatArrayBatchedCopy_alignedK_contig
33
+ 3.448 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
34
+ 3.387 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
35
+ 3.280 ms _scatter_gather_elementwise_kernel
36
+ 3.245 ms multi_tensor_swizzle_row_scaling_kernel
37
+ 3.006 ms multi_tensor_swizzle_col_scaling_kernel
38
+ 2.630 ms fused_rope_backward_kernel
39
+ 2.465 ms fused_rope_forward_kernel
40
+ 2.351 ms triton_red_fused__to_copy_mul_silu_sum_1
41
+ 2.227 ms swizzle_col_scaling_kernel
42
+ 2.056 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
43
+
44
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 128 (64/iter)
46
+ reduce_grad sum : 9.215 ms/iter
47
+ reduce_grad busy (union) : 9.215 ms/iter
48
+ ... overlapped with compute: 8.600 ms/iter ( 93.3 %)
49
+ ... exposed on critical path: 0.615 ms/iter ( 6.7 %)
50
+ other GPU work (union) : 304.595 ms/iter
51
+ traced wall : 362.827 ms/iter
52
+ --> time hidden by overlap : 8.600 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 104.142 ms elementwise_kernel
56
+ 73.412 ms device_kernel
57
+ 60.052 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
58
+ 48.238 ms multi_tensor_apply_kernel
59
+ 47.684 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
60
+ 43.994 ms cast_mxfp8_2D_kernel
61
+ 41.995 ms vectorized_elementwise_kernel
62
+ 24.387 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
63
+ 23.745 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
64
+ 16.678 ms unrolled_elementwise_kernel
65
+ 11.859 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
66
+ REDUCE 11.637 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
67
+ 10.707 ms ncclDevKernel_AllGather_RING_LL
68
+ 8.219 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
69
+ 7.883 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
70
+ 7.670 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
71
+ 6.984 ms triton_poi_fused__to_copy_cat_0
72
+ REDUCE 6.545 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
73
+ 5.756 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
74
+ 5.641 ms setGroupedGemmWgradArguments
75
+ 5.491 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
76
+ 4.983 ms gatherTopK
77
+ 4.719 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
78
+ 4.479 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
79
+ 4.266 ms reduce_kernel
80
+ 4.030 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
81
+ 3.785 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
82
+ 3.768 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
83
+ 3.680 ms rmsnorm_bwd_tuned_kernel
84
+ 3.473 ms CatArrayBatchedCopy_alignedK_contig
85
+ 3.345 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
86
+ 3.290 ms _scatter_gather_elementwise_kernel
87
+ 3.242 ms multi_tensor_swizzle_row_scaling_kernel
88
+ 3.128 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
89
+ 2.989 ms multi_tensor_swizzle_col_scaling_kernel
90
+ 2.949 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
91
+ 2.626 ms fused_rope_backward_kernel
92
+ 2.455 ms fused_rope_forward_kernel
93
+ 2.367 ms triton_red_fused__to_copy_mul_silu_sum_1
94
+ 2.256 ms swizzle_col_scaling_kernel
95
+
96
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 128 (64/iter)
98
+ reduce_grad sum : 9.091 ms/iter
99
+ reduce_grad busy (union) : 9.091 ms/iter
100
+ ... overlapped with compute: 8.477 ms/iter ( 93.2 %)
101
+ ... exposed on critical path: 0.614 ms/iter ( 6.8 %)
102
+ other GPU work (union) : 306.324 ms/iter
103
+ traced wall : 407.153 ms/iter
104
+ --> time hidden by overlap : 8.477 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 104.106 ms elementwise_kernel
108
+ 74.640 ms device_kernel
109
+ 59.997 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
110
+ 49.081 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
111
+ 48.255 ms multi_tensor_apply_kernel
112
+ 43.595 ms cast_mxfp8_2D_kernel
113
+ 39.678 ms vectorized_elementwise_kernel
114
+ 24.151 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
115
+ 23.808 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
116
+ 16.711 ms unrolled_elementwise_kernel
117
+ REDUCE 11.705 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
118
+ 10.891 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
119
+ 10.771 ms ncclDevKernel_AllGather_RING_LL
120
+ 8.251 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
121
+ 8.007 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
122
+ 7.711 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
123
+ 7.003 ms triton_poi_fused__to_copy_cat_0
124
+ REDUCE 6.458 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
125
+ 6.252 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
126
+ 5.604 ms setGroupedGemmWgradArguments
127
+ 4.997 ms gatherTopK
128
+ 4.771 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
129
+ 4.523 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
130
+ 4.222 ms reduce_kernel
131
+ 4.128 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
132
+ 3.973 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
133
+ 3.895 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
134
+ 3.780 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
135
+ 3.705 ms rmsnorm_bwd_tuned_kernel
136
+ 3.506 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
137
+ 3.464 ms CatArrayBatchedCopy_alignedK_contig
138
+ 3.387 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
139
+ 3.282 ms _scatter_gather_elementwise_kernel
140
+ 3.222 ms multi_tensor_swizzle_row_scaling_kernel
141
+ 2.995 ms multi_tensor_swizzle_col_scaling_kernel
142
+ 2.630 ms fused_rope_backward_kernel
143
+ 2.461 ms fused_rope_forward_kernel
144
+ 2.346 ms triton_red_fused__to_copy_mul_silu_sum_1
145
+ 2.225 ms swizzle_col_scaling_kernel
146
+ 2.015 ms triton_poi_fused__to_copy_mul_silu_0
147
+
148
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 128 (64/iter)
150
+ reduce_grad sum : 9.082 ms/iter
151
+ reduce_grad busy (union) : 9.082 ms/iter
152
+ ... overlapped with compute: 8.467 ms/iter ( 93.2 %)
153
+ ... exposed on critical path: 0.615 ms/iter ( 6.8 %)
154
+ other GPU work (union) : 304.705 ms/iter
155
+ traced wall : 362.800 ms/iter
156
+ --> time hidden by overlap : 8.467 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 104.219 ms elementwise_kernel
160
+ 74.904 ms device_kernel
161
+ 60.025 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
162
+ 49.022 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
163
+ 48.228 ms multi_tensor_apply_kernel
164
+ 43.637 ms cast_mxfp8_2D_kernel
165
+ 39.885 ms vectorized_elementwise_kernel
166
+ 24.080 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
167
+ 23.387 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
168
+ 16.721 ms unrolled_elementwise_kernel
169
+ REDUCE 11.160 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
170
+ 10.756 ms ncclDevKernel_AllGather_RING_LL
171
+ 9.843 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
172
+ 8.314 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
173
+ 7.956 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
174
+ 7.815 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
175
+ 7.008 ms triton_poi_fused__to_copy_cat_0
176
+ REDUCE 6.628 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
177
+ 5.811 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
178
+ 5.673 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
179
+ 5.654 ms setGroupedGemmWgradArguments
180
+ 4.976 ms gatherTopK
181
+ 4.804 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
182
+ 4.538 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
183
+ 4.208 ms reduce_kernel
184
+ 3.995 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
185
+ 3.972 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
186
+ 3.805 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
187
+ 3.651 ms rmsnorm_bwd_tuned_kernel
188
+ 3.464 ms CatArrayBatchedCopy_alignedK_contig
189
+ 3.426 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
190
+ 3.278 ms _scatter_gather_elementwise_kernel
191
+ 3.240 ms multi_tensor_swizzle_row_scaling_kernel
192
+ 2.999 ms multi_tensor_swizzle_col_scaling_kernel
193
+ 2.997 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
194
+ 2.625 ms fused_rope_backward_kernel
195
+ 2.459 ms fused_rope_forward_kernel
196
+ 2.356 ms triton_red_fused__to_copy_mul_silu_sum_1
197
+ 2.241 ms swizzle_col_scaling_kernel
198
+ 2.235 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
199
+
200
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 128 (64/iter)
202
+ reduce_grad sum : 8.894 ms/iter
203
+ reduce_grad busy (union) : 8.894 ms/iter
204
+ ... overlapped with compute: 8.288 ms/iter ( 93.2 %)
205
+ ... exposed on critical path: 0.606 ms/iter ( 6.8 %)
206
+ other GPU work (union) : 305.557 ms/iter
207
+ traced wall : 362.873 ms/iter
208
+ --> time hidden by overlap : 8.288 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 104.178 ms elementwise_kernel
212
+ 74.395 ms device_kernel
213
+ 59.972 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
214
+ 48.653 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
215
+ 48.248 ms multi_tensor_apply_kernel
216
+ 43.654 ms cast_mxfp8_2D_kernel
217
+ 39.865 ms vectorized_elementwise_kernel
218
+ 23.922 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
219
+ 23.872 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
220
+ 16.611 ms unrolled_elementwise_kernel
221
+ REDUCE 11.630 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
222
+ 10.538 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
223
+ 8.898 ms ncclDevKernel_AllGather_RING_LL
224
+ 8.274 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
225
+ 7.923 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
226
+ 7.764 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
227
+ 7.000 ms triton_poi_fused__to_copy_cat_0
228
+ REDUCE 6.429 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
229
+ 6.110 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
230
+ 5.481 ms setGroupedGemmWgradArguments
231
+ 4.981 ms gatherTopK
232
+ 4.802 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
233
+ 4.536 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
234
+ 4.236 ms reduce_kernel
235
+ 4.017 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
236
+ 3.852 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
237
+ 3.794 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
238
+ 3.672 ms rmsnorm_bwd_tuned_kernel
239
+ 3.476 ms CatArrayBatchedCopy_alignedK_contig
240
+ 3.377 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
241
+ 3.285 ms _scatter_gather_elementwise_kernel
242
+ 3.240 ms multi_tensor_swizzle_row_scaling_kernel
243
+ 3.182 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
244
+ 3.007 ms multi_tensor_swizzle_col_scaling_kernel
245
+ 2.631 ms fused_rope_backward_kernel
246
+ 2.501 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
247
+ 2.464 ms fused_rope_forward_kernel
248
+ 2.347 ms triton_red_fused__to_copy_mul_silu_sum_1
249
+ 2.214 ms swizzle_col_scaling_kernel
250
+ 2.013 ms triton_poi_fused__to_copy_mul_silu_0
251
+
252
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 128 (64/iter)
254
+ reduce_grad sum : 9.029 ms/iter
255
+ reduce_grad busy (union) : 9.029 ms/iter
256
+ ... overlapped with compute: 8.401 ms/iter ( 93.0 %)
257
+ ... exposed on critical path: 0.628 ms/iter ( 7.0 %)
258
+ other GPU work (union) : 302.106 ms/iter
259
+ traced wall : 362.882 ms/iter
260
+ --> time hidden by overlap : 8.401 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 104.221 ms elementwise_kernel
264
+ 73.951 ms device_kernel
265
+ 59.986 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
266
+ 48.217 ms multi_tensor_apply_kernel
267
+ 48.136 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
268
+ 43.626 ms cast_mxfp8_2D_kernel
269
+ 39.881 ms vectorized_elementwise_kernel
270
+ 23.870 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
271
+ 23.704 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
272
+ 16.713 ms unrolled_elementwise_kernel
273
+ 11.556 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
274
+ REDUCE 11.451 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
275
+ 8.772 ms ncclDevKernel_AllGather_RING_LL
276
+ 8.273 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
277
+ 7.952 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
278
+ 7.750 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
279
+ 7.003 ms triton_poi_fused__to_copy_cat_0
280
+ REDUCE 6.542 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
281
+ 5.850 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
282
+ 5.627 ms setGroupedGemmWgradArguments
283
+ 5.304 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
284
+ 4.964 ms gatherTopK
285
+ 4.773 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
286
+ 4.531 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
287
+ 4.258 ms reduce_kernel
288
+ 3.994 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
289
+ 3.897 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
290
+ 3.781 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
291
+ 3.680 ms rmsnorm_bwd_tuned_kernel
292
+ 3.466 ms CatArrayBatchedCopy_alignedK_contig
293
+ 3.397 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
294
+ 3.301 ms _scatter_gather_elementwise_kernel
295
+ 3.198 ms multi_tensor_swizzle_row_scaling_kernel
296
+ 2.998 ms multi_tensor_swizzle_col_scaling_kernel
297
+ 2.835 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
298
+ 2.776 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
299
+ 2.627 ms fused_rope_backward_kernel
300
+ 2.460 ms fused_rope_forward_kernel
301
+ 2.353 ms triton_red_fused__to_copy_mul_silu_sum_1
302
+ 2.224 ms swizzle_col_scaling_kernel
303
+
304
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 128 (64/iter)
306
+ reduce_grad sum : 8.997 ms/iter
307
+ reduce_grad busy (union) : 8.997 ms/iter
308
+ ... overlapped with compute: 8.369 ms/iter ( 93.0 %)
309
+ ... exposed on critical path: 0.628 ms/iter ( 7.0 %)
310
+ other GPU work (union) : 304.289 ms/iter
311
+ traced wall : 362.570 ms/iter
312
+ --> time hidden by overlap : 8.369 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 104.108 ms elementwise_kernel
316
+ 74.193 ms device_kernel
317
+ 60.031 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
318
+ 48.600 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
319
+ 48.179 ms multi_tensor_apply_kernel
320
+ 44.105 ms cast_mxfp8_2D_kernel
321
+ 39.947 ms vectorized_elementwise_kernel
322
+ 24.076 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
323
+ 23.692 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
324
+ 16.692 ms unrolled_elementwise_kernel
325
+ REDUCE 11.704 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
326
+ 10.582 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
327
+ 8.933 ms ncclDevKernel_AllGather_RING_LL
328
+ 8.332 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
329
+ 8.087 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
330
+ 7.839 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
331
+ 7.009 ms triton_poi_fused__to_copy_cat_0
332
+ REDUCE 6.400 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
333
+ 5.703 ms setGroupedGemmWgradArguments
334
+ 5.653 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
335
+ 5.177 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
336
+ 4.989 ms gatherTopK
337
+ 4.802 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
338
+ 4.542 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
339
+ 4.274 ms reduce_kernel
340
+ 4.031 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
341
+ 4.012 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
342
+ 3.846 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
343
+ 3.693 ms rmsnorm_bwd_tuned_kernel
344
+ 3.469 ms CatArrayBatchedCopy_alignedK_contig
345
+ 3.412 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
346
+ 3.284 ms _scatter_gather_elementwise_kernel
347
+ 3.216 ms multi_tensor_swizzle_row_scaling_kernel
348
+ 3.005 ms multi_tensor_swizzle_col_scaling_kernel
349
+ 2.938 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
350
+ 2.631 ms fused_rope_backward_kernel
351
+ 2.480 ms fused_rope_forward_kernel
352
+ 2.365 ms triton_red_fused__to_copy_mul_silu_sum_1
353
+ 2.252 ms swizzle_col_scaling_kernel
354
+ 2.112 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
355
+
356
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 128 (64/iter)
358
+ reduce_grad sum : 9.052 ms/iter
359
+ reduce_grad busy (union) : 9.052 ms/iter
360
+ ... overlapped with compute: 8.448 ms/iter ( 93.3 %)
361
+ ... exposed on critical path: 0.604 ms/iter ( 6.7 %)
362
+ other GPU work (union) : 304.397 ms/iter
363
+ traced wall : 362.648 ms/iter
364
+ --> time hidden by overlap : 8.448 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 104.128 ms elementwise_kernel
368
+ 73.310 ms device_kernel
369
+ 59.971 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
370
+ 48.187 ms multi_tensor_apply_kernel
371
+ 47.868 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
372
+ 44.117 ms cast_mxfp8_2D_kernel
373
+ 39.906 ms vectorized_elementwise_kernel
374
+ 23.914 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
375
+ 23.718 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
376
+ 16.678 ms unrolled_elementwise_kernel
377
+ 12.742 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
378
+ REDUCE 11.820 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
379
+ 11.113 ms ncclDevKernel_AllGather_RING_LL
380
+ 8.232 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
381
+ 7.915 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
382
+ 7.784 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
383
+ 6.986 ms triton_poi_fused__to_copy_cat_0
384
+ REDUCE 6.644 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
385
+ 6.334 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
386
+ 5.636 ms setGroupedGemmWgradArguments
387
+ 5.243 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
388
+ 4.973 ms gatherTopK
389
+ 4.789 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
390
+ 4.499 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
391
+ 4.266 ms reduce_kernel
392
+ 4.024 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
393
+ 3.842 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
394
+ 3.816 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
395
+ 3.724 ms rmsnorm_bwd_tuned_kernel
396
+ 3.469 ms CatArrayBatchedCopy_alignedK_contig
397
+ 3.371 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
398
+ 3.298 ms _scatter_gather_elementwise_kernel
399
+ 3.263 ms multi_tensor_swizzle_row_scaling_kernel
400
+ 3.181 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
401
+ 2.995 ms multi_tensor_swizzle_col_scaling_kernel
402
+ 2.628 ms fused_rope_backward_kernel
403
+ 2.450 ms fused_rope_forward_kernel
404
+ 2.364 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
405
+ 2.362 ms triton_red_fused__to_copy_mul_silu_sum_1
406
+ 2.252 ms swizzle_col_scaling_kernel
407
+
408
+ === G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 128 (64/iter)
410
+ reduce_grad sum : 9.232 ms/iter
411
+ reduce_grad busy (union) : 9.232 ms/iter
412
+ ... overlapped with compute: 8.620 ms/iter ( 93.4 %)
413
+ ... exposed on critical path: 0.612 ms/iter ( 6.6 %)
414
+ other GPU work (union) : 306.052 ms/iter
415
+ traced wall : 362.662 ms/iter
416
+ --> time hidden by overlap : 8.620 ms/iter
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=9.074 ms/iter overlap=8.459 ms/iter (93.2%) exposed=0.615 ms/iter status=PASS
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log CHANGED
The diff for this file is too large to render. See raw diff
 
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
2
+ kernels : 157909
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv CHANGED
@@ -5,3 +5,6 @@ G3_K8 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/ti
5
  G4_K2 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
6
  G4_K4 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
7
  G4_K6 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
 
 
 
 
5
  G4_K2 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
6
  G4_K4 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
7
  G4_K6 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
8
+ G4_K8 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
9
+ G4_K10 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
10
+ G4_K16 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json