granularity results granularity_next_b200/tiera_topk_trace/moonep/tierbtrace @ 2026-09-21T06:19:34Z from sp-aiinfra-interactive-aghosh-8-mb99k-8xb200-moe-cg-0
Browse files- .gitattributes +3 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +105 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env +105 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +3 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +9 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt +416 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt +1 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +0 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log +6 -0
- granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv +3 -0
.gitattributes
CHANGED
|
@@ -106,3 +106,6 @@ granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G3_K8_
|
|
| 106 |
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 107 |
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 108 |
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 107 |
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 108 |
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 109 |
+
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 110 |
+
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
| 111 |
+
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9bf77de49b53024ab595177d714037b276d691b7da6944ed42ddf794f157259f
|
| 3 |
+
size 13358738
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"devices": 8,
|
| 3 |
+
"reduce_busy_ms_per_iteration": 9.808375,
|
| 4 |
+
"overlapped_ms_per_iteration": 9.171625,
|
| 5 |
+
"exposed_ms_per_iteration": 0.6365000000000001,
|
| 6 |
+
"overlap_pct": 93.50809894605376,
|
| 7 |
+
"minimum_overlap_pct": 0.1,
|
| 8 |
+
"status": "PASS"
|
| 9 |
+
}
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
=== kernels on device 0, by total time ===
|
| 3 |
+
124.743 ms elementwise_kernel
|
| 4 |
+
87.460 ms device_kernel
|
| 5 |
+
59.517 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 6 |
+
48.986 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 7 |
+
48.137 ms multi_tensor_apply_kernel
|
| 8 |
+
47.454 ms cast_mxfp8_2D_kernel
|
| 9 |
+
42.268 ms vectorized_elementwise_kernel
|
| 10 |
+
25.040 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 11 |
+
23.857 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 12 |
+
16.600 ms unrolled_elementwise_kernel
|
| 13 |
+
REDUCE 14.560 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 14 |
+
14.287 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 15 |
+
10.740 ms ncclDevKernel_AllGather_RING_LL
|
| 16 |
+
9.253 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 17 |
+
8.507 ms triton_poi_fused__to_copy_cat_0
|
| 18 |
+
8.264 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 19 |
+
8.238 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 20 |
+
7.766 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 21 |
+
REDUCE 6.528 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 22 |
+
5.946 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 23 |
+
5.532 ms setGroupedGemmWgradArguments
|
| 24 |
+
5.057 ms gatherTopK
|
| 25 |
+
4.984 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 26 |
+
4.804 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 27 |
+
4.532 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 28 |
+
4.223 ms reduce_kernel
|
| 29 |
+
4.047 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 30 |
+
3.901 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 31 |
+
3.854 ms rmsnorm_bwd_tuned_kernel
|
| 32 |
+
3.782 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 33 |
+
3.663 ms multi_tensor_swizzle_row_scaling_kernel
|
| 34 |
+
3.564 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 35 |
+
3.450 ms CatArrayBatchedCopy_alignedK_contig
|
| 36 |
+
3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 37 |
+
3.370 ms _scatter_gather_elementwise_kernel
|
| 38 |
+
3.297 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 39 |
+
3.055 ms multi_tensor_swizzle_col_scaling_kernel
|
| 40 |
+
2.631 ms fused_rope_backward_kernel
|
| 41 |
+
2.558 ms fused_rope_forward_kernel
|
| 42 |
+
2.434 ms triton_poi_fused__to_copy_mul_silu_0
|
| 43 |
+
|
| 44 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
|
| 45 |
+
reduce_grad kernels : 128 (64/iter)
|
| 46 |
+
reduce_grad sum : 10.544 ms/iter
|
| 47 |
+
reduce_grad busy (union) : 10.544 ms/iter
|
| 48 |
+
... overlapped with compute: 9.888 ms/iter ( 93.8 %)
|
| 49 |
+
... exposed on critical path: 0.656 ms/iter ( 6.2 %)
|
| 50 |
+
other GPU work (union) : 331.413 ms/iter
|
| 51 |
+
traced wall : 391.824 ms/iter
|
| 52 |
+
--> time hidden by overlap : 9.888 ms/iter
|
| 53 |
+
|
| 54 |
+
=== kernels on device 1, by total time ===
|
| 55 |
+
124.677 ms elementwise_kernel
|
| 56 |
+
86.754 ms device_kernel
|
| 57 |
+
59.448 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 58 |
+
49.352 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 59 |
+
48.224 ms multi_tensor_apply_kernel
|
| 60 |
+
47.782 ms cast_mxfp8_2D_kernel
|
| 61 |
+
42.326 ms vectorized_elementwise_kernel
|
| 62 |
+
25.141 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 63 |
+
23.750 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 64 |
+
16.683 ms unrolled_elementwise_kernel
|
| 65 |
+
REDUCE 14.215 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 66 |
+
13.948 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 67 |
+
12.621 ms ncclDevKernel_AllGather_RING_LL
|
| 68 |
+
9.583 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 69 |
+
9.433 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 70 |
+
8.489 ms triton_poi_fused__to_copy_cat_0
|
| 71 |
+
8.208 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 72 |
+
8.143 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 73 |
+
7.661 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 74 |
+
REDUCE 6.497 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 75 |
+
6.372 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 76 |
+
5.575 ms setGroupedGemmWgradArguments
|
| 77 |
+
5.163 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 78 |
+
5.064 ms gatherTopK
|
| 79 |
+
4.714 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 80 |
+
4.477 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 81 |
+
4.228 ms reduce_kernel
|
| 82 |
+
4.125 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 83 |
+
3.850 ms rmsnorm_bwd_tuned_kernel
|
| 84 |
+
3.782 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 85 |
+
3.761 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 86 |
+
3.704 ms multi_tensor_swizzle_row_scaling_kernel
|
| 87 |
+
3.517 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 88 |
+
3.474 ms CatArrayBatchedCopy_alignedK_contig
|
| 89 |
+
3.385 ms _scatter_gather_elementwise_kernel
|
| 90 |
+
3.338 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 91 |
+
3.303 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 92 |
+
3.028 ms multi_tensor_swizzle_col_scaling_kernel
|
| 93 |
+
2.632 ms fused_rope_backward_kernel
|
| 94 |
+
2.538 ms fused_rope_forward_kernel
|
| 95 |
+
|
| 96 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
|
| 97 |
+
reduce_grad kernels : 128 (64/iter)
|
| 98 |
+
reduce_grad sum : 10.356 ms/iter
|
| 99 |
+
reduce_grad busy (union) : 10.356 ms/iter
|
| 100 |
+
... overlapped with compute: 9.710 ms/iter ( 93.8 %)
|
| 101 |
+
... exposed on critical path: 0.646 ms/iter ( 6.2 %)
|
| 102 |
+
other GPU work (union) : 337.144 ms/iter
|
| 103 |
+
traced wall : 392.464 ms/iter
|
| 104 |
+
--> time hidden by overlap : 9.710 ms/iter
|
| 105 |
+
|
| 106 |
+
=== kernels on device 2, by total time ===
|
| 107 |
+
124.523 ms elementwise_kernel
|
| 108 |
+
88.064 ms device_kernel
|
| 109 |
+
59.370 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 110 |
+
53.586 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 111 |
+
48.233 ms multi_tensor_apply_kernel
|
| 112 |
+
47.355 ms cast_mxfp8_2D_kernel
|
| 113 |
+
42.019 ms vectorized_elementwise_kernel
|
| 114 |
+
24.373 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 115 |
+
23.815 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 116 |
+
16.659 ms unrolled_elementwise_kernel
|
| 117 |
+
REDUCE 12.377 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 118 |
+
10.193 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 119 |
+
10.056 ms ncclDevKernel_AllGather_RING_LL
|
| 120 |
+
8.815 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 121 |
+
8.506 ms triton_poi_fused__to_copy_cat_0
|
| 122 |
+
8.249 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 123 |
+
8.006 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 124 |
+
7.905 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 125 |
+
7.707 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 126 |
+
6.728 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 127 |
+
REDUCE 6.381 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 128 |
+
5.551 ms setGroupedGemmWgradArguments
|
| 129 |
+
5.062 ms gatherTopK
|
| 130 |
+
4.755 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 131 |
+
4.743 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 132 |
+
4.515 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 133 |
+
4.186 ms reduce_kernel
|
| 134 |
+
4.086 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 135 |
+
3.895 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 136 |
+
3.795 ms rmsnorm_bwd_tuned_kernel
|
| 137 |
+
3.790 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 138 |
+
3.694 ms multi_tensor_swizzle_row_scaling_kernel
|
| 139 |
+
3.687 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 140 |
+
3.465 ms CatArrayBatchedCopy_alignedK_contig
|
| 141 |
+
3.396 ms _scatter_gather_elementwise_kernel
|
| 142 |
+
3.385 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 143 |
+
3.298 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 144 |
+
3.054 ms multi_tensor_swizzle_col_scaling_kernel
|
| 145 |
+
2.633 ms fused_rope_backward_kernel
|
| 146 |
+
2.468 ms fused_rope_forward_kernel
|
| 147 |
+
|
| 148 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
|
| 149 |
+
reduce_grad kernels : 128 (64/iter)
|
| 150 |
+
reduce_grad sum : 9.379 ms/iter
|
| 151 |
+
reduce_grad busy (union) : 9.379 ms/iter
|
| 152 |
+
... overlapped with compute: 8.751 ms/iter ( 93.3 %)
|
| 153 |
+
... exposed on critical path: 0.627 ms/iter ( 6.7 %)
|
| 154 |
+
other GPU work (union) : 335.373 ms/iter
|
| 155 |
+
traced wall : 392.194 ms/iter
|
| 156 |
+
--> time hidden by overlap : 8.751 ms/iter
|
| 157 |
+
|
| 158 |
+
=== kernels on device 3, by total time ===
|
| 159 |
+
124.615 ms elementwise_kernel
|
| 160 |
+
88.174 ms device_kernel
|
| 161 |
+
59.359 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 162 |
+
54.730 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 163 |
+
48.247 ms multi_tensor_apply_kernel
|
| 164 |
+
47.377 ms cast_mxfp8_2D_kernel
|
| 165 |
+
42.251 ms vectorized_elementwise_kernel
|
| 166 |
+
24.065 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 167 |
+
23.780 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 168 |
+
16.663 ms unrolled_elementwise_kernel
|
| 169 |
+
REDUCE 11.624 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 170 |
+
9.701 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 171 |
+
8.513 ms triton_poi_fused__to_copy_cat_0
|
| 172 |
+
8.506 ms ncclDevKernel_AllGather_RING_LL
|
| 173 |
+
8.478 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 174 |
+
8.313 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 175 |
+
7.941 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 176 |
+
7.807 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 177 |
+
7.328 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 178 |
+
REDUCE 6.537 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 179 |
+
6.219 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 180 |
+
5.513 ms setGroupedGemmWgradArguments
|
| 181 |
+
5.073 ms gatherTopK
|
| 182 |
+
4.791 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 183 |
+
4.601 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 184 |
+
4.532 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 185 |
+
4.188 ms reduce_kernel
|
| 186 |
+
4.101 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 187 |
+
3.971 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 188 |
+
3.820 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 189 |
+
3.729 ms rmsnorm_bwd_tuned_kernel
|
| 190 |
+
3.652 ms multi_tensor_swizzle_row_scaling_kernel
|
| 191 |
+
3.462 ms CatArrayBatchedCopy_alignedK_contig
|
| 192 |
+
3.426 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 193 |
+
3.406 ms _scatter_gather_elementwise_kernel
|
| 194 |
+
3.297 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 195 |
+
3.233 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 196 |
+
3.058 ms multi_tensor_swizzle_col_scaling_kernel
|
| 197 |
+
2.636 ms fused_rope_backward_kernel
|
| 198 |
+
2.464 ms fused_rope_forward_kernel
|
| 199 |
+
|
| 200 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
|
| 201 |
+
reduce_grad kernels : 128 (64/iter)
|
| 202 |
+
reduce_grad sum : 9.081 ms/iter
|
| 203 |
+
reduce_grad busy (union) : 9.081 ms/iter
|
| 204 |
+
... overlapped with compute: 8.463 ms/iter ( 93.2 %)
|
| 205 |
+
... exposed on critical path: 0.618 ms/iter ( 6.8 %)
|
| 206 |
+
other GPU work (union) : 334.445 ms/iter
|
| 207 |
+
traced wall : 391.299 ms/iter
|
| 208 |
+
--> time hidden by overlap : 8.463 ms/iter
|
| 209 |
+
|
| 210 |
+
=== kernels on device 4, by total time ===
|
| 211 |
+
124.595 ms elementwise_kernel
|
| 212 |
+
87.627 ms device_kernel
|
| 213 |
+
59.359 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 214 |
+
54.267 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 215 |
+
48.141 ms multi_tensor_apply_kernel
|
| 216 |
+
47.381 ms cast_mxfp8_2D_kernel
|
| 217 |
+
41.984 ms vectorized_elementwise_kernel
|
| 218 |
+
24.408 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 219 |
+
23.866 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 220 |
+
16.683 ms unrolled_elementwise_kernel
|
| 221 |
+
REDUCE 12.149 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 222 |
+
12.049 ms ncclDevKernel_AllGather_RING_LL
|
| 223 |
+
10.317 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 224 |
+
8.499 ms triton_poi_fused__to_copy_cat_0
|
| 225 |
+
8.407 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 226 |
+
8.276 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 227 |
+
7.996 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 228 |
+
7.764 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 229 |
+
6.676 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 230 |
+
REDUCE 6.459 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 231 |
+
5.776 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 232 |
+
5.526 ms setGroupedGemmWgradArguments
|
| 233 |
+
5.078 ms gatherTopK
|
| 234 |
+
4.780 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 235 |
+
4.529 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 236 |
+
4.519 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 237 |
+
4.197 ms reduce_kernel
|
| 238 |
+
4.118 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 239 |
+
3.937 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 240 |
+
3.843 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 241 |
+
3.806 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 242 |
+
3.756 ms rmsnorm_bwd_tuned_kernel
|
| 243 |
+
3.667 ms multi_tensor_swizzle_row_scaling_kernel
|
| 244 |
+
3.469 ms CatArrayBatchedCopy_alignedK_contig
|
| 245 |
+
3.377 ms _scatter_gather_elementwise_kernel
|
| 246 |
+
3.367 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 247 |
+
3.302 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 248 |
+
3.023 ms multi_tensor_swizzle_col_scaling_kernel
|
| 249 |
+
2.626 ms fused_rope_backward_kernel
|
| 250 |
+
2.466 ms fused_rope_forward_kernel
|
| 251 |
+
|
| 252 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
|
| 253 |
+
reduce_grad kernels : 128 (64/iter)
|
| 254 |
+
reduce_grad sum : 9.304 ms/iter
|
| 255 |
+
reduce_grad busy (union) : 9.304 ms/iter
|
| 256 |
+
... overlapped with compute: 8.669 ms/iter ( 93.2 %)
|
| 257 |
+
... exposed on critical path: 0.635 ms/iter ( 6.8 %)
|
| 258 |
+
other GPU work (union) : 335.415 ms/iter
|
| 259 |
+
traced wall : 392.463 ms/iter
|
| 260 |
+
--> time hidden by overlap : 8.669 ms/iter
|
| 261 |
+
|
| 262 |
+
=== kernels on device 5, by total time ===
|
| 263 |
+
124.633 ms elementwise_kernel
|
| 264 |
+
87.722 ms device_kernel
|
| 265 |
+
59.344 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 266 |
+
53.161 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 267 |
+
48.273 ms multi_tensor_apply_kernel
|
| 268 |
+
47.349 ms cast_mxfp8_2D_kernel
|
| 269 |
+
42.052 ms vectorized_elementwise_kernel
|
| 270 |
+
24.002 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 271 |
+
23.894 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 272 |
+
16.733 ms unrolled_elementwise_kernel
|
| 273 |
+
11.804 ms ncclDevKernel_AllGather_RING_LL
|
| 274 |
+
REDUCE 11.686 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 275 |
+
11.387 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 276 |
+
9.915 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 277 |
+
9.385 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 278 |
+
8.507 ms triton_poi_fused__to_copy_cat_0
|
| 279 |
+
8.274 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 280 |
+
7.948 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 281 |
+
7.769 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 282 |
+
REDUCE 6.511 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 283 |
+
5.519 ms setGroupedGemmWgradArguments
|
| 284 |
+
5.378 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 285 |
+
5.180 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 286 |
+
5.067 ms gatherTopK
|
| 287 |
+
4.770 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 288 |
+
4.521 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 289 |
+
4.216 ms reduce_kernel
|
| 290 |
+
4.095 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 291 |
+
3.893 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 292 |
+
3.781 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 293 |
+
3.747 ms rmsnorm_bwd_tuned_kernel
|
| 294 |
+
3.595 ms multi_tensor_swizzle_row_scaling_kernel
|
| 295 |
+
3.467 ms CatArrayBatchedCopy_alignedK_contig
|
| 296 |
+
3.399 ms _scatter_gather_elementwise_kernel
|
| 297 |
+
3.394 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 298 |
+
3.302 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 299 |
+
3.051 ms multi_tensor_swizzle_col_scaling_kernel
|
| 300 |
+
2.935 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 301 |
+
2.635 ms fused_rope_backward_kernel
|
| 302 |
+
2.460 ms fused_rope_forward_kernel
|
| 303 |
+
|
| 304 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
|
| 305 |
+
reduce_grad kernels : 128 (64/iter)
|
| 306 |
+
reduce_grad sum : 9.099 ms/iter
|
| 307 |
+
reduce_grad busy (union) : 9.099 ms/iter
|
| 308 |
+
... overlapped with compute: 8.478 ms/iter ( 93.2 %)
|
| 309 |
+
... exposed on critical path: 0.621 ms/iter ( 6.8 %)
|
| 310 |
+
other GPU work (union) : 337.042 ms/iter
|
| 311 |
+
traced wall : 392.314 ms/iter
|
| 312 |
+
--> time hidden by overlap : 8.478 ms/iter
|
| 313 |
+
|
| 314 |
+
=== kernels on device 6, by total time ===
|
| 315 |
+
124.699 ms elementwise_kernel
|
| 316 |
+
87.979 ms device_kernel
|
| 317 |
+
59.423 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 318 |
+
50.071 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 319 |
+
48.190 ms multi_tensor_apply_kernel
|
| 320 |
+
47.864 ms cast_mxfp8_2D_kernel
|
| 321 |
+
42.256 ms vectorized_elementwise_kernel
|
| 322 |
+
24.570 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 323 |
+
24.074 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 324 |
+
16.660 ms unrolled_elementwise_kernel
|
| 325 |
+
REDUCE 14.139 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 326 |
+
11.896 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 327 |
+
11.593 ms ncclDevKernel_AllGather_RING_LL
|
| 328 |
+
9.074 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 329 |
+
8.843 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 330 |
+
8.510 ms triton_poi_fused__to_copy_cat_0
|
| 331 |
+
8.312 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 332 |
+
8.236 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 333 |
+
7.832 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 334 |
+
REDUCE 6.448 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 335 |
+
6.282 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 336 |
+
5.603 ms setGroupedGemmWgradArguments
|
| 337 |
+
5.070 ms gatherTopK
|
| 338 |
+
4.916 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 339 |
+
4.787 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 340 |
+
4.544 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 341 |
+
4.215 ms reduce_kernel
|
| 342 |
+
4.128 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 343 |
+
4.009 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 344 |
+
3.914 ms rmsnorm_bwd_tuned_kernel
|
| 345 |
+
3.822 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 346 |
+
3.680 ms multi_tensor_swizzle_row_scaling_kernel
|
| 347 |
+
3.584 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 348 |
+
3.465 ms CatArrayBatchedCopy_alignedK_contig
|
| 349 |
+
3.411 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 350 |
+
3.410 ms _scatter_gather_elementwise_kernel
|
| 351 |
+
3.304 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 352 |
+
3.053 ms multi_tensor_swizzle_col_scaling_kernel
|
| 353 |
+
2.638 ms fused_rope_backward_kernel
|
| 354 |
+
2.579 ms fused_rope_forward_kernel
|
| 355 |
+
|
| 356 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
|
| 357 |
+
reduce_grad kernels : 128 (64/iter)
|
| 358 |
+
reduce_grad sum : 10.294 ms/iter
|
| 359 |
+
reduce_grad busy (union) : 10.294 ms/iter
|
| 360 |
+
... overlapped with compute: 9.655 ms/iter ( 93.8 %)
|
| 361 |
+
... exposed on critical path: 0.638 ms/iter ( 6.2 %)
|
| 362 |
+
other GPU work (union) : 336.380 ms/iter
|
| 363 |
+
traced wall : 391.953 ms/iter
|
| 364 |
+
--> time hidden by overlap : 9.655 ms/iter
|
| 365 |
+
|
| 366 |
+
=== kernels on device 7, by total time ===
|
| 367 |
+
124.623 ms elementwise_kernel
|
| 368 |
+
86.503 ms device_kernel
|
| 369 |
+
59.404 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o3604484096i64i64_tensorptri32gmemalign16o409601_tensorptri32_0
|
| 370 |
+
49.040 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o3604484096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o409601_tensorptri32gm_0
|
| 371 |
+
48.184 ms multi_tensor_apply_kernel
|
| 372 |
+
47.912 ms cast_mxfp8_2D_kernel
|
| 373 |
+
42.122 ms vectorized_elementwise_kernel
|
| 374 |
+
25.215 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 375 |
+
23.721 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 376 |
+
16.655 ms unrolled_elementwise_kernel
|
| 377 |
+
16.409 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 378 |
+
REDUCE 14.094 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 379 |
+
10.327 ms ncclDevKernel_AllGather_RING_LL
|
| 380 |
+
9.497 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tensor_0
|
| 381 |
+
8.490 ms triton_poi_fused__to_copy_cat_0
|
| 382 |
+
8.216 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 383 |
+
8.143 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 384 |
+
7.796 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 385 |
+
7.786 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 386 |
+
REDUCE 6.726 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 387 |
+
5.624 ms setGroupedGemmWgradArguments
|
| 388 |
+
5.126 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o450244096i64i64_tensorptri32gmemalign16o1350721_tensorptri32gmemalign16o450241_tens_0
|
| 389 |
+
5.058 ms gatherTopK
|
| 390 |
+
4.853 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 391 |
+
4.769 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 392 |
+
4.502 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 393 |
+
4.216 ms reduce_kernel
|
| 394 |
+
4.127 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_409601_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 395 |
+
3.869 ms rmsnorm_bwd_tuned_kernel
|
| 396 |
+
3.838 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 397 |
+
3.825 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 398 |
+
3.666 ms multi_tensor_swizzle_row_scaling_kernel
|
| 399 |
+
3.472 ms CatArrayBatchedCopy_alignedK_contig
|
| 400 |
+
3.395 ms _scatter_gather_elementwise_kernel
|
| 401 |
+
3.363 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 402 |
+
3.303 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 403 |
+
3.040 ms multi_tensor_swizzle_col_scaling_kernel
|
| 404 |
+
3.002 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 405 |
+
2.630 ms fused_rope_backward_kernel
|
| 406 |
+
2.554 ms fused_rope_forward_kernel
|
| 407 |
+
|
| 408 |
+
=== G4_K10__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
|
| 409 |
+
reduce_grad kernels : 128 (64/iter)
|
| 410 |
+
reduce_grad sum : 10.410 ms/iter
|
| 411 |
+
reduce_grad busy (union) : 10.410 ms/iter
|
| 412 |
+
... overlapped with compute: 9.759 ms/iter ( 93.7 %)
|
| 413 |
+
... exposed on critical path: 0.651 ms/iter ( 6.3 %)
|
| 414 |
+
other GPU work (union) : 335.222 ms/iter
|
| 415 |
+
traced wall : 391.638 ms/iter
|
| 416 |
+
--> time hidden by overlap : 9.759 ms/iter
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
devices=8 reduce=9.808 ms/iter overlap=9.172 ms/iter (93.5%) exposed=0.637 ms/iter status=PASS
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=G4_K10
|
| 3 |
+
family=tierA
|
| 4 |
+
source_model=
|
| 5 |
+
standard_num_layers=
|
| 6 |
+
base_rung=G4
|
| 7 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 8 |
+
branch=moonep_backend_async_grad_reduce
|
| 9 |
+
commit=2e68c8ad
|
| 10 |
+
moonep_source_dir=/workspace/MoonEP
|
| 11 |
+
moonep_source_branch=megatron-ddp-symmetric-params
|
| 12 |
+
moonep_source_commit=d8bc666
|
| 13 |
+
num_layers=16
|
| 14 |
+
hidden=4096
|
| 15 |
+
ffn_hidden=14336
|
| 16 |
+
moe_ffn_hidden=896
|
| 17 |
+
num_experts=128
|
| 18 |
+
topk=10
|
| 19 |
+
experts_per_rank=16
|
| 20 |
+
expert_params=11010048
|
| 21 |
+
heads=32
|
| 22 |
+
kv_groups=8
|
| 23 |
+
kv_channels=
|
| 24 |
+
qk_layernorm=0
|
| 25 |
+
shared_ffn=0
|
| 26 |
+
shared_expert_overlap=0
|
| 27 |
+
moe_token_dispatcher=flex
|
| 28 |
+
mla=0
|
| 29 |
+
q_lora_rank=
|
| 30 |
+
kv_lora_rank=512
|
| 31 |
+
qk_head_dim=128
|
| 32 |
+
qk_pos_emb_head_dim=64
|
| 33 |
+
v_head_dim=128
|
| 34 |
+
dense_first_layers=0
|
| 35 |
+
moe_layers=16
|
| 36 |
+
rotary_base=1000000
|
| 37 |
+
rotary_percent=1.0
|
| 38 |
+
rotary_scaling=
|
| 39 |
+
mscale=
|
| 40 |
+
mscale_all_dim=
|
| 41 |
+
qkv_bias=0
|
| 42 |
+
router_balancing=aux_loss
|
| 43 |
+
aux_loss_coeff=1e-2
|
| 44 |
+
router_pre_softmax=0
|
| 45 |
+
router_fusion=0
|
| 46 |
+
router_dtype=fp32
|
| 47 |
+
router_groups=
|
| 48 |
+
router_group_topk=
|
| 49 |
+
router_score_function=softmax
|
| 50 |
+
router_expert_bias=0
|
| 51 |
+
router_bias_update_rate=
|
| 52 |
+
router_topk_scaling_factor=
|
| 53 |
+
shared_expert_gate=0
|
| 54 |
+
apply_layernorm_1p=0
|
| 55 |
+
attention_output_gate=0
|
| 56 |
+
linear_attention_type=
|
| 57 |
+
linear_attention_freq=
|
| 58 |
+
linear_conv_kernel_dim=
|
| 59 |
+
linear_key_head_dim=
|
| 60 |
+
linear_value_head_dim=
|
| 61 |
+
linear_num_key_heads=
|
| 62 |
+
linear_num_value_heads=
|
| 63 |
+
mtp_num_layers=0
|
| 64 |
+
mtp_loss_scaling_factor=0.1
|
| 65 |
+
no_weight_decay_cond_type=
|
| 66 |
+
max_position_embeddings=32768
|
| 67 |
+
init_method_std=0.01
|
| 68 |
+
device_initiated_grouped_gemm=1
|
| 69 |
+
norm_epsilon=1e-5
|
| 70 |
+
seq_length=4096
|
| 71 |
+
micro_batch_size=1
|
| 72 |
+
global_batch_size=16
|
| 73 |
+
num_microbatches=2
|
| 74 |
+
assignments_per_rank=40960
|
| 75 |
+
mean_tokens_per_expert=2560
|
| 76 |
+
slot_policy=matched
|
| 77 |
+
slots_per_rank=16
|
| 78 |
+
capacity_mode=none
|
| 79 |
+
capacity=none
|
| 80 |
+
cuda_graph=1
|
| 81 |
+
probe_mode=0
|
| 82 |
+
async_grad_reduce=1
|
| 83 |
+
branch_role=timing
|
| 84 |
+
memory_dump_mode=none
|
| 85 |
+
memory_snapshot_path=
|
| 86 |
+
cuda_graph_warmup_steps=3
|
| 87 |
+
recompute=none
|
| 88 |
+
force_balance=0
|
| 89 |
+
train_iters=10
|
| 90 |
+
ep_size=8
|
| 91 |
+
echo_dispatch_overlap=1
|
| 92 |
+
echo_dispatch_recompute=0
|
| 93 |
+
ultraep_quota_min_tokens=
|
| 94 |
+
ultraep_weight_sync_join=
|
| 95 |
+
profile_enable=1
|
| 96 |
+
kernel_trace=1
|
| 97 |
+
fp8_mode=mxfp8
|
| 98 |
+
fp8_recipe=mxfp8
|
| 99 |
+
training_seed=1234
|
| 100 |
+
learning_rate=1e-4
|
| 101 |
+
min_learning_rate=1.0e-5
|
| 102 |
+
lr_warmup_iters=5
|
| 103 |
+
state_dump_enable=0
|
| 104 |
+
state_dump_dir=
|
| 105 |
+
moonep_assert_no_pending_reduce=1
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
report : /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
|
| 2 |
+
kernels : 157894
|
| 3 |
+
devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
|
| 4 |
+
processes: 1
|
| 5 |
+
markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
|
| 6 |
+
PASS: all ranks present and backend markers emitted
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e2507c3838c92981cdfd7ecf187636d7cbc642492e01ecd6b8e154d381670cf8
|
| 3 |
+
size 13450239
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"devices": 8,
|
| 3 |
+
"reduce_busy_ms_per_iteration": 9.724499999999999,
|
| 4 |
+
"overlapped_ms_per_iteration": 9.09,
|
| 5 |
+
"exposed_ms_per_iteration": 0.6345000000000001,
|
| 6 |
+
"overlap_pct": 93.47524294308191,
|
| 7 |
+
"minimum_overlap_pct": 0.1,
|
| 8 |
+
"status": "PASS"
|
| 9 |
+
}
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
=== kernels on device 0, by total time ===
|
| 3 |
+
185.953 ms elementwise_kernel
|
| 4 |
+
130.258 ms device_kernel
|
| 5 |
+
60.883 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 6 |
+
58.596 ms cast_mxfp8_2D_kernel
|
| 7 |
+
48.858 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 8 |
+
48.675 ms vectorized_elementwise_kernel
|
| 9 |
+
48.263 ms multi_tensor_apply_kernel
|
| 10 |
+
24.913 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 11 |
+
23.952 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 12 |
+
23.884 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 13 |
+
16.632 ms unrolled_elementwise_kernel
|
| 14 |
+
13.739 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 15 |
+
13.032 ms triton_poi_fused__to_copy_cat_0
|
| 16 |
+
REDUCE 12.732 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 17 |
+
12.410 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 18 |
+
11.694 ms ncclDevKernel_AllGather_RING_LL
|
| 19 |
+
8.275 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 20 |
+
8.078 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 21 |
+
7.765 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 22 |
+
REDUCE 6.779 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 23 |
+
5.716 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 24 |
+
5.235 ms gatherTopK
|
| 25 |
+
5.077 ms setGroupedGemmWgradArguments
|
| 26 |
+
4.890 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 27 |
+
4.796 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 28 |
+
4.662 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 29 |
+
4.546 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 30 |
+
4.219 ms reduce_kernel
|
| 31 |
+
4.191 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 32 |
+
4.180 ms multi_tensor_swizzle_row_scaling_kernel
|
| 33 |
+
3.911 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 34 |
+
3.780 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 35 |
+
3.768 ms rmsnorm_bwd_tuned_kernel
|
| 36 |
+
3.696 ms _scatter_gather_elementwise_kernel
|
| 37 |
+
3.672 ms triton_poi_fused__to_copy_mul_silu_0
|
| 38 |
+
3.455 ms CatArrayBatchedCopy_alignedK_contig
|
| 39 |
+
3.380 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 40 |
+
3.175 ms multi_tensor_swizzle_col_scaling_kernel
|
| 41 |
+
3.080 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 42 |
+
2.628 ms fused_rope_backward_kernel
|
| 43 |
+
|
| 44 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
|
| 45 |
+
reduce_grad kernels : 128 (64/iter)
|
| 46 |
+
reduce_grad sum : 9.755 ms/iter
|
| 47 |
+
reduce_grad busy (union) : 9.755 ms/iter
|
| 48 |
+
... overlapped with compute: 9.130 ms/iter ( 93.6 %)
|
| 49 |
+
... exposed on critical path: 0.625 ms/iter ( 6.4 %)
|
| 50 |
+
other GPU work (union) : 410.945 ms/iter
|
| 51 |
+
traced wall : 487.279 ms/iter
|
| 52 |
+
--> time hidden by overlap : 9.130 ms/iter
|
| 53 |
+
|
| 54 |
+
=== kernels on device 1, by total time ===
|
| 55 |
+
185.813 ms elementwise_kernel
|
| 56 |
+
128.677 ms device_kernel
|
| 57 |
+
60.727 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 58 |
+
58.881 ms cast_mxfp8_2D_kernel
|
| 59 |
+
48.818 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 60 |
+
48.790 ms vectorized_elementwise_kernel
|
| 61 |
+
48.201 ms multi_tensor_apply_kernel
|
| 62 |
+
25.032 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 63 |
+
24.216 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 64 |
+
23.815 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 65 |
+
16.747 ms unrolled_elementwise_kernel
|
| 66 |
+
13.772 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 67 |
+
13.492 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 68 |
+
13.000 ms triton_poi_fused__to_copy_cat_0
|
| 69 |
+
REDUCE 12.626 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 70 |
+
8.383 ms ncclDevKernel_AllGather_RING_LL
|
| 71 |
+
8.209 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 72 |
+
7.972 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 73 |
+
7.645 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 74 |
+
REDUCE 6.845 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 75 |
+
5.941 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 76 |
+
5.221 ms gatherTopK
|
| 77 |
+
5.036 ms setGroupedGemmWgradArguments
|
| 78 |
+
4.900 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 79 |
+
4.731 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 80 |
+
4.473 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 81 |
+
4.229 ms reduce_kernel
|
| 82 |
+
4.217 ms multi_tensor_swizzle_row_scaling_kernel
|
| 83 |
+
4.217 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 84 |
+
3.778 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 85 |
+
3.768 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 86 |
+
3.740 ms _scatter_gather_elementwise_kernel
|
| 87 |
+
3.705 ms rmsnorm_bwd_tuned_kernel
|
| 88 |
+
3.661 ms triton_poi_fused__to_copy_mul_silu_0
|
| 89 |
+
3.466 ms CatArrayBatchedCopy_alignedK_contig
|
| 90 |
+
3.403 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 91 |
+
3.391 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 92 |
+
3.339 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 93 |
+
3.126 ms multi_tensor_swizzle_col_scaling_kernel
|
| 94 |
+
2.629 ms fused_rope_backward_kernel
|
| 95 |
+
|
| 96 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
|
| 97 |
+
reduce_grad kernels : 128 (64/iter)
|
| 98 |
+
reduce_grad sum : 9.736 ms/iter
|
| 99 |
+
reduce_grad busy (union) : 9.736 ms/iter
|
| 100 |
+
... overlapped with compute: 9.111 ms/iter ( 93.6 %)
|
| 101 |
+
... exposed on critical path: 0.625 ms/iter ( 6.4 %)
|
| 102 |
+
other GPU work (union) : 408.592 ms/iter
|
| 103 |
+
traced wall : 487.269 ms/iter
|
| 104 |
+
--> time hidden by overlap : 9.111 ms/iter
|
| 105 |
+
|
| 106 |
+
=== kernels on device 2, by total time ===
|
| 107 |
+
185.818 ms elementwise_kernel
|
| 108 |
+
130.847 ms device_kernel
|
| 109 |
+
60.764 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 110 |
+
58.518 ms cast_mxfp8_2D_kernel
|
| 111 |
+
50.710 ms vectorized_elementwise_kernel
|
| 112 |
+
48.898 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 113 |
+
48.203 ms multi_tensor_apply_kernel
|
| 114 |
+
25.222 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 115 |
+
24.035 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 116 |
+
23.886 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 117 |
+
16.677 ms unrolled_elementwise_kernel
|
| 118 |
+
13.868 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 119 |
+
13.031 ms triton_poi_fused__to_copy_cat_0
|
| 120 |
+
REDUCE 12.681 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 121 |
+
11.355 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 122 |
+
8.933 ms ncclDevKernel_AllGather_RING_LL
|
| 123 |
+
8.250 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 124 |
+
8.120 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 125 |
+
7.708 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 126 |
+
REDUCE 6.756 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 127 |
+
6.103 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 128 |
+
5.234 ms gatherTopK
|
| 129 |
+
4.987 ms setGroupedGemmWgradArguments
|
| 130 |
+
4.899 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 131 |
+
4.878 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 132 |
+
4.782 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 133 |
+
4.533 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 134 |
+
4.273 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 135 |
+
4.194 ms reduce_kernel
|
| 136 |
+
4.173 ms multi_tensor_swizzle_row_scaling_kernel
|
| 137 |
+
3.905 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 138 |
+
3.778 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 139 |
+
3.729 ms rmsnorm_bwd_tuned_kernel
|
| 140 |
+
3.725 ms _scatter_gather_elementwise_kernel
|
| 141 |
+
3.669 ms triton_poi_fused__to_copy_mul_silu_0
|
| 142 |
+
3.461 ms CatArrayBatchedCopy_alignedK_contig
|
| 143 |
+
3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 144 |
+
3.186 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 145 |
+
3.162 ms multi_tensor_swizzle_col_scaling_kernel
|
| 146 |
+
2.628 ms fused_rope_backward_kernel
|
| 147 |
+
|
| 148 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
|
| 149 |
+
reduce_grad kernels : 128 (64/iter)
|
| 150 |
+
reduce_grad sum : 9.718 ms/iter
|
| 151 |
+
reduce_grad busy (union) : 9.718 ms/iter
|
| 152 |
+
... overlapped with compute: 9.073 ms/iter ( 93.4 %)
|
| 153 |
+
... exposed on critical path: 0.645 ms/iter ( 6.6 %)
|
| 154 |
+
other GPU work (union) : 410.582 ms/iter
|
| 155 |
+
traced wall : 530.417 ms/iter
|
| 156 |
+
--> time hidden by overlap : 9.073 ms/iter
|
| 157 |
+
|
| 158 |
+
=== kernels on device 3, by total time ===
|
| 159 |
+
185.985 ms elementwise_kernel
|
| 160 |
+
131.612 ms device_kernel
|
| 161 |
+
60.724 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 162 |
+
58.618 ms cast_mxfp8_2D_kernel
|
| 163 |
+
49.566 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 164 |
+
48.779 ms vectorized_elementwise_kernel
|
| 165 |
+
48.214 ms multi_tensor_apply_kernel
|
| 166 |
+
24.563 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 167 |
+
24.194 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 168 |
+
23.389 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 169 |
+
16.705 ms unrolled_elementwise_kernel
|
| 170 |
+
13.570 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 171 |
+
13.039 ms triton_poi_fused__to_copy_cat_0
|
| 172 |
+
REDUCE 12.524 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 173 |
+
10.127 ms ncclDevKernel_AllGather_RING_LL
|
| 174 |
+
9.674 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 175 |
+
8.315 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 176 |
+
8.114 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 177 |
+
7.812 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 178 |
+
REDUCE 6.816 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 179 |
+
5.911 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 180 |
+
5.217 ms gatherTopK
|
| 181 |
+
4.998 ms setGroupedGemmWgradArguments
|
| 182 |
+
4.918 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 183 |
+
4.808 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 184 |
+
4.547 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 185 |
+
4.287 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 186 |
+
4.179 ms multi_tensor_swizzle_row_scaling_kernel
|
| 187 |
+
4.177 ms reduce_kernel
|
| 188 |
+
3.967 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 189 |
+
3.827 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 190 |
+
3.740 ms _scatter_gather_elementwise_kernel
|
| 191 |
+
3.681 ms rmsnorm_bwd_tuned_kernel
|
| 192 |
+
3.665 ms triton_poi_fused__to_copy_mul_silu_0
|
| 193 |
+
3.470 ms CatArrayBatchedCopy_alignedK_contig
|
| 194 |
+
3.421 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 195 |
+
3.347 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 196 |
+
3.163 ms multi_tensor_swizzle_col_scaling_kernel
|
| 197 |
+
2.635 ms fused_rope_backward_kernel
|
| 198 |
+
2.466 ms fused_rope_forward_kernel
|
| 199 |
+
|
| 200 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
|
| 201 |
+
reduce_grad kernels : 128 (64/iter)
|
| 202 |
+
reduce_grad sum : 9.670 ms/iter
|
| 203 |
+
reduce_grad busy (union) : 9.670 ms/iter
|
| 204 |
+
... overlapped with compute: 9.028 ms/iter ( 93.4 %)
|
| 205 |
+
... exposed on critical path: 0.642 ms/iter ( 6.6 %)
|
| 206 |
+
other GPU work (union) : 408.736 ms/iter
|
| 207 |
+
traced wall : 487.194 ms/iter
|
| 208 |
+
--> time hidden by overlap : 9.028 ms/iter
|
| 209 |
+
|
| 210 |
+
=== kernels on device 4, by total time ===
|
| 211 |
+
185.905 ms elementwise_kernel
|
| 212 |
+
129.675 ms device_kernel
|
| 213 |
+
60.748 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 214 |
+
58.671 ms cast_mxfp8_2D_kernel
|
| 215 |
+
50.666 ms vectorized_elementwise_kernel
|
| 216 |
+
48.255 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 217 |
+
48.204 ms multi_tensor_apply_kernel
|
| 218 |
+
25.777 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 219 |
+
24.097 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 220 |
+
23.903 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 221 |
+
16.642 ms unrolled_elementwise_kernel
|
| 222 |
+
14.072 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 223 |
+
13.014 ms triton_poi_fused__to_copy_cat_0
|
| 224 |
+
REDUCE 12.736 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 225 |
+
11.153 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 226 |
+
9.216 ms ncclDevKernel_AllGather_RING_LL
|
| 227 |
+
8.264 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 228 |
+
8.074 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 229 |
+
7.771 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 230 |
+
6.940 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 231 |
+
REDUCE 6.753 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 232 |
+
5.260 ms gatherTopK
|
| 233 |
+
5.087 ms setGroupedGemmWgradArguments
|
| 234 |
+
4.893 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 235 |
+
4.809 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 236 |
+
4.549 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 237 |
+
4.274 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 238 |
+
4.200 ms multi_tensor_swizzle_row_scaling_kernel
|
| 239 |
+
4.195 ms reduce_kernel
|
| 240 |
+
3.854 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 241 |
+
3.792 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 242 |
+
3.746 ms rmsnorm_bwd_tuned_kernel
|
| 243 |
+
3.694 ms _scatter_gather_elementwise_kernel
|
| 244 |
+
3.667 ms triton_poi_fused__to_copy_mul_silu_0
|
| 245 |
+
3.470 ms CatArrayBatchedCopy_alignedK_contig
|
| 246 |
+
3.364 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 247 |
+
3.218 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 248 |
+
3.155 ms multi_tensor_swizzle_col_scaling_kernel
|
| 249 |
+
2.934 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 250 |
+
2.632 ms fused_rope_backward_kernel
|
| 251 |
+
|
| 252 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
|
| 253 |
+
reduce_grad kernels : 128 (64/iter)
|
| 254 |
+
reduce_grad sum : 9.745 ms/iter
|
| 255 |
+
reduce_grad busy (union) : 9.745 ms/iter
|
| 256 |
+
... overlapped with compute: 9.104 ms/iter ( 93.4 %)
|
| 257 |
+
... exposed on critical path: 0.641 ms/iter ( 6.6 %)
|
| 258 |
+
other GPU work (union) : 409.825 ms/iter
|
| 259 |
+
traced wall : 529.382 ms/iter
|
| 260 |
+
--> time hidden by overlap : 9.104 ms/iter
|
| 261 |
+
|
| 262 |
+
=== kernels on device 5, by total time ===
|
| 263 |
+
185.961 ms elementwise_kernel
|
| 264 |
+
130.127 ms device_kernel
|
| 265 |
+
60.748 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 266 |
+
58.582 ms cast_mxfp8_2D_kernel
|
| 267 |
+
48.736 ms vectorized_elementwise_kernel
|
| 268 |
+
48.616 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 269 |
+
48.259 ms multi_tensor_apply_kernel
|
| 270 |
+
25.310 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 271 |
+
23.960 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 272 |
+
23.649 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 273 |
+
16.814 ms unrolled_elementwise_kernel
|
| 274 |
+
13.936 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 275 |
+
13.029 ms triton_poi_fused__to_copy_cat_0
|
| 276 |
+
REDUCE 12.509 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 277 |
+
12.045 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 278 |
+
11.218 ms ncclDevKernel_AllGather_RING_LL
|
| 279 |
+
8.262 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 280 |
+
8.080 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 281 |
+
7.759 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 282 |
+
REDUCE 6.862 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 283 |
+
5.662 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 284 |
+
5.464 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 285 |
+
5.229 ms gatherTopK
|
| 286 |
+
5.056 ms setGroupedGemmWgradArguments
|
| 287 |
+
4.902 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 288 |
+
4.776 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 289 |
+
4.531 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 290 |
+
4.266 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 291 |
+
4.211 ms reduce_kernel
|
| 292 |
+
4.152 ms multi_tensor_swizzle_row_scaling_kernel
|
| 293 |
+
3.896 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 294 |
+
3.786 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 295 |
+
3.730 ms rmsnorm_bwd_tuned_kernel
|
| 296 |
+
3.721 ms _scatter_gather_elementwise_kernel
|
| 297 |
+
3.671 ms triton_poi_fused__to_copy_mul_silu_0
|
| 298 |
+
3.464 ms CatArrayBatchedCopy_alignedK_contig
|
| 299 |
+
3.391 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 300 |
+
3.154 ms multi_tensor_swizzle_col_scaling_kernel
|
| 301 |
+
2.983 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 302 |
+
2.626 ms fused_rope_backward_kernel
|
| 303 |
+
|
| 304 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
|
| 305 |
+
reduce_grad kernels : 128 (64/iter)
|
| 306 |
+
reduce_grad sum : 9.686 ms/iter
|
| 307 |
+
reduce_grad busy (union) : 9.686 ms/iter
|
| 308 |
+
... overlapped with compute: 9.045 ms/iter ( 93.4 %)
|
| 309 |
+
... exposed on critical path: 0.641 ms/iter ( 6.6 %)
|
| 310 |
+
other GPU work (union) : 410.918 ms/iter
|
| 311 |
+
traced wall : 487.069 ms/iter
|
| 312 |
+
--> time hidden by overlap : 9.045 ms/iter
|
| 313 |
+
|
| 314 |
+
=== kernels on device 6, by total time ===
|
| 315 |
+
185.903 ms elementwise_kernel
|
| 316 |
+
130.934 ms device_kernel
|
| 317 |
+
60.720 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 318 |
+
59.068 ms cast_mxfp8_2D_kernel
|
| 319 |
+
50.826 ms vectorized_elementwise_kernel
|
| 320 |
+
49.246 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 321 |
+
48.191 ms multi_tensor_apply_kernel
|
| 322 |
+
24.750 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 323 |
+
24.170 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 324 |
+
23.382 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 325 |
+
16.682 ms unrolled_elementwise_kernel
|
| 326 |
+
13.654 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 327 |
+
13.021 ms triton_poi_fused__to_copy_cat_0
|
| 328 |
+
REDUCE 12.749 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 329 |
+
9.270 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 330 |
+
8.888 ms ncclDevKernel_AllGather_RING_LL
|
| 331 |
+
8.333 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 332 |
+
8.148 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 333 |
+
7.821 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 334 |
+
REDUCE 6.774 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 335 |
+
6.714 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 336 |
+
5.219 ms gatherTopK
|
| 337 |
+
5.145 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 338 |
+
5.099 ms setGroupedGemmWgradArguments
|
| 339 |
+
4.911 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 340 |
+
4.828 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 341 |
+
4.575 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 342 |
+
4.233 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 343 |
+
4.219 ms reduce_kernel
|
| 344 |
+
4.202 ms multi_tensor_swizzle_row_scaling_kernel
|
| 345 |
+
4.021 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 346 |
+
3.831 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 347 |
+
3.774 ms rmsnorm_bwd_tuned_kernel
|
| 348 |
+
3.722 ms _scatter_gather_elementwise_kernel
|
| 349 |
+
3.659 ms triton_poi_fused__to_copy_mul_silu_0
|
| 350 |
+
3.461 ms CatArrayBatchedCopy_alignedK_contig
|
| 351 |
+
3.403 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 352 |
+
3.338 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 353 |
+
3.157 ms multi_tensor_swizzle_col_scaling_kernel
|
| 354 |
+
2.633 ms fused_rope_backward_kernel
|
| 355 |
+
|
| 356 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
|
| 357 |
+
reduce_grad kernels : 128 (64/iter)
|
| 358 |
+
reduce_grad sum : 9.761 ms/iter
|
| 359 |
+
reduce_grad busy (union) : 9.761 ms/iter
|
| 360 |
+
... overlapped with compute: 9.129 ms/iter ( 93.5 %)
|
| 361 |
+
... exposed on critical path: 0.632 ms/iter ( 6.5 %)
|
| 362 |
+
other GPU work (union) : 410.856 ms/iter
|
| 363 |
+
traced wall : 529.363 ms/iter
|
| 364 |
+
--> time hidden by overlap : 9.129 ms/iter
|
| 365 |
+
|
| 366 |
+
=== kernels on device 7, by total time ===
|
| 367 |
+
185.800 ms elementwise_kernel
|
| 368 |
+
128.396 ms device_kernel
|
| 369 |
+
60.718 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5570564096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0
|
| 370 |
+
59.059 ms cast_mxfp8_2D_kernel
|
| 371 |
+
50.852 ms vectorized_elementwise_kernel
|
| 372 |
+
48.800 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5570564096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0
|
| 373 |
+
48.163 ms multi_tensor_apply_kernel
|
| 374 |
+
25.135 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tensor_0
|
| 375 |
+
23.793 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 376 |
+
23.782 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 377 |
+
16.650 ms unrolled_elementwise_kernel
|
| 378 |
+
13.861 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o696004096i64i64_tensorptri32gmemalign16o2088001_tensorptri32gmemalign16o696001_tens_0
|
| 379 |
+
13.304 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 380 |
+
13.007 ms triton_poi_fused__to_copy_cat_0
|
| 381 |
+
REDUCE 12.573 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 382 |
+
10.835 ms ncclDevKernel_AllGather_RING_LL
|
| 383 |
+
8.228 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 384 |
+
8.056 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 385 |
+
7.804 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 386 |
+
REDUCE 6.878 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 387 |
+
6.274 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 388 |
+
5.954 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 389 |
+
5.241 ms gatherTopK
|
| 390 |
+
5.147 ms setGroupedGemmWgradArguments
|
| 391 |
+
4.898 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 392 |
+
4.798 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 393 |
+
4.521 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 394 |
+
4.230 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 395 |
+
4.225 ms reduce_kernel
|
| 396 |
+
4.217 ms multi_tensor_swizzle_row_scaling_kernel
|
| 397 |
+
3.843 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 398 |
+
3.823 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 399 |
+
3.761 ms _scatter_gather_elementwise_kernel
|
| 400 |
+
3.706 ms rmsnorm_bwd_tuned_kernel
|
| 401 |
+
3.649 ms triton_poi_fused__to_copy_mul_silu_0
|
| 402 |
+
3.464 ms CatArrayBatchedCopy_alignedK_contig
|
| 403 |
+
3.424 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 404 |
+
3.357 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 405 |
+
3.141 ms multi_tensor_swizzle_col_scaling_kernel
|
| 406 |
+
2.632 ms fused_rope_backward_kernel
|
| 407 |
+
|
| 408 |
+
=== G4_K16__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
|
| 409 |
+
reduce_grad kernels : 128 (64/iter)
|
| 410 |
+
reduce_grad sum : 9.725 ms/iter
|
| 411 |
+
reduce_grad busy (union) : 9.725 ms/iter
|
| 412 |
+
... overlapped with compute: 9.100 ms/iter ( 93.6 %)
|
| 413 |
+
... exposed on critical path: 0.625 ms/iter ( 6.4 %)
|
| 414 |
+
other GPU work (union) : 412.420 ms/iter
|
| 415 |
+
traced wall : 530.283 ms/iter
|
| 416 |
+
--> time hidden by overlap : 9.100 ms/iter
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
devices=8 reduce=9.724 ms/iter overlap=9.090 ms/iter (93.5%) exposed=0.635 ms/iter status=PASS
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
backend=moonep
|
| 2 |
+
config=G4_K16
|
| 3 |
+
family=tierA
|
| 4 |
+
source_model=
|
| 5 |
+
standard_num_layers=
|
| 6 |
+
base_rung=G4
|
| 7 |
+
repo=/workspace/Megatron-MoE-Echo
|
| 8 |
+
branch=moonep_backend_async_grad_reduce
|
| 9 |
+
commit=2e68c8ad
|
| 10 |
+
moonep_source_dir=/workspace/MoonEP
|
| 11 |
+
moonep_source_branch=megatron-ddp-symmetric-params
|
| 12 |
+
moonep_source_commit=d8bc666
|
| 13 |
+
num_layers=16
|
| 14 |
+
hidden=4096
|
| 15 |
+
ffn_hidden=14336
|
| 16 |
+
moe_ffn_hidden=896
|
| 17 |
+
num_experts=128
|
| 18 |
+
topk=16
|
| 19 |
+
experts_per_rank=16
|
| 20 |
+
expert_params=11010048
|
| 21 |
+
heads=32
|
| 22 |
+
kv_groups=8
|
| 23 |
+
kv_channels=
|
| 24 |
+
qk_layernorm=0
|
| 25 |
+
shared_ffn=0
|
| 26 |
+
shared_expert_overlap=0
|
| 27 |
+
moe_token_dispatcher=flex
|
| 28 |
+
mla=0
|
| 29 |
+
q_lora_rank=
|
| 30 |
+
kv_lora_rank=512
|
| 31 |
+
qk_head_dim=128
|
| 32 |
+
qk_pos_emb_head_dim=64
|
| 33 |
+
v_head_dim=128
|
| 34 |
+
dense_first_layers=0
|
| 35 |
+
moe_layers=16
|
| 36 |
+
rotary_base=1000000
|
| 37 |
+
rotary_percent=1.0
|
| 38 |
+
rotary_scaling=
|
| 39 |
+
mscale=
|
| 40 |
+
mscale_all_dim=
|
| 41 |
+
qkv_bias=0
|
| 42 |
+
router_balancing=aux_loss
|
| 43 |
+
aux_loss_coeff=1e-2
|
| 44 |
+
router_pre_softmax=0
|
| 45 |
+
router_fusion=0
|
| 46 |
+
router_dtype=fp32
|
| 47 |
+
router_groups=
|
| 48 |
+
router_group_topk=
|
| 49 |
+
router_score_function=softmax
|
| 50 |
+
router_expert_bias=0
|
| 51 |
+
router_bias_update_rate=
|
| 52 |
+
router_topk_scaling_factor=
|
| 53 |
+
shared_expert_gate=0
|
| 54 |
+
apply_layernorm_1p=0
|
| 55 |
+
attention_output_gate=0
|
| 56 |
+
linear_attention_type=
|
| 57 |
+
linear_attention_freq=
|
| 58 |
+
linear_conv_kernel_dim=
|
| 59 |
+
linear_key_head_dim=
|
| 60 |
+
linear_value_head_dim=
|
| 61 |
+
linear_num_key_heads=
|
| 62 |
+
linear_num_value_heads=
|
| 63 |
+
mtp_num_layers=0
|
| 64 |
+
mtp_loss_scaling_factor=0.1
|
| 65 |
+
no_weight_decay_cond_type=
|
| 66 |
+
max_position_embeddings=32768
|
| 67 |
+
init_method_std=0.01
|
| 68 |
+
device_initiated_grouped_gemm=1
|
| 69 |
+
norm_epsilon=1e-5
|
| 70 |
+
seq_length=4096
|
| 71 |
+
micro_batch_size=1
|
| 72 |
+
global_batch_size=16
|
| 73 |
+
num_microbatches=2
|
| 74 |
+
assignments_per_rank=65536
|
| 75 |
+
mean_tokens_per_expert=4096
|
| 76 |
+
slot_policy=matched
|
| 77 |
+
slots_per_rank=16
|
| 78 |
+
capacity_mode=none
|
| 79 |
+
capacity=none
|
| 80 |
+
cuda_graph=1
|
| 81 |
+
probe_mode=0
|
| 82 |
+
async_grad_reduce=1
|
| 83 |
+
branch_role=timing
|
| 84 |
+
memory_dump_mode=none
|
| 85 |
+
memory_snapshot_path=
|
| 86 |
+
cuda_graph_warmup_steps=3
|
| 87 |
+
recompute=none
|
| 88 |
+
force_balance=0
|
| 89 |
+
train_iters=10
|
| 90 |
+
ep_size=8
|
| 91 |
+
echo_dispatch_overlap=1
|
| 92 |
+
echo_dispatch_recompute=0
|
| 93 |
+
ultraep_quota_min_tokens=
|
| 94 |
+
ultraep_weight_sync_join=
|
| 95 |
+
profile_enable=1
|
| 96 |
+
kernel_trace=1
|
| 97 |
+
fp8_mode=mxfp8
|
| 98 |
+
fp8_recipe=mxfp8
|
| 99 |
+
training_seed=1234
|
| 100 |
+
learning_rate=1e-4
|
| 101 |
+
min_learning_rate=1.0e-5
|
| 102 |
+
lr_warmup_iters=5
|
| 103 |
+
state_dump_enable=0
|
| 104 |
+
state_dump_dir=
|
| 105 |
+
moonep_assert_no_pending_reduce=1
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
report : /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
|
| 2 |
+
kernels : 157954
|
| 3 |
+
devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
|
| 4 |
+
processes: 1
|
| 5 |
+
markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
|
| 6 |
+
PASS: all ranks present and backend markers emitted
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bc07abf7c635833117d47053bae06cc1a7ce369d15d022463debc970eca5bbff
|
| 3 |
+
size 13044490
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"devices": 8,
|
| 3 |
+
"reduce_busy_ms_per_iteration": 9.074,
|
| 4 |
+
"overlapped_ms_per_iteration": 8.45875,
|
| 5 |
+
"exposed_ms_per_iteration": 0.61525,
|
| 6 |
+
"overlap_pct": 93.21963852766144,
|
| 7 |
+
"minimum_overlap_pct": 0.1,
|
| 8 |
+
"status": "PASS"
|
| 9 |
+
}
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
=== kernels on device 0, by total time ===
|
| 3 |
+
104.215 ms elementwise_kernel
|
| 4 |
+
73.573 ms device_kernel
|
| 5 |
+
60.108 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 6 |
+
48.226 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 7 |
+
48.201 ms multi_tensor_apply_kernel
|
| 8 |
+
43.680 ms cast_mxfp8_2D_kernel
|
| 9 |
+
39.886 ms vectorized_elementwise_kernel
|
| 10 |
+
23.964 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 11 |
+
23.859 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 12 |
+
16.662 ms unrolled_elementwise_kernel
|
| 13 |
+
REDUCE 11.863 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 14 |
+
11.483 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 15 |
+
11.263 ms ncclDevKernel_AllGather_RING_LL
|
| 16 |
+
8.275 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 17 |
+
7.997 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 18 |
+
7.762 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 19 |
+
7.008 ms triton_poi_fused__to_copy_cat_0
|
| 20 |
+
REDUCE 6.568 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 21 |
+
6.406 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 22 |
+
5.652 ms setGroupedGemmWgradArguments
|
| 23 |
+
4.999 ms gatherTopK
|
| 24 |
+
4.796 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 25 |
+
4.536 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 26 |
+
4.216 ms reduce_kernel
|
| 27 |
+
3.931 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 28 |
+
3.900 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 29 |
+
3.779 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 30 |
+
3.679 ms rmsnorm_bwd_tuned_kernel
|
| 31 |
+
3.525 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 32 |
+
3.454 ms CatArrayBatchedCopy_alignedK_contig
|
| 33 |
+
3.448 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 34 |
+
3.387 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 35 |
+
3.280 ms _scatter_gather_elementwise_kernel
|
| 36 |
+
3.245 ms multi_tensor_swizzle_row_scaling_kernel
|
| 37 |
+
3.006 ms multi_tensor_swizzle_col_scaling_kernel
|
| 38 |
+
2.630 ms fused_rope_backward_kernel
|
| 39 |
+
2.465 ms fused_rope_forward_kernel
|
| 40 |
+
2.351 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 41 |
+
2.227 ms swizzle_col_scaling_kernel
|
| 42 |
+
2.056 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 43 |
+
|
| 44 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) ===
|
| 45 |
+
reduce_grad kernels : 128 (64/iter)
|
| 46 |
+
reduce_grad sum : 9.215 ms/iter
|
| 47 |
+
reduce_grad busy (union) : 9.215 ms/iter
|
| 48 |
+
... overlapped with compute: 8.600 ms/iter ( 93.3 %)
|
| 49 |
+
... exposed on critical path: 0.615 ms/iter ( 6.7 %)
|
| 50 |
+
other GPU work (union) : 304.595 ms/iter
|
| 51 |
+
traced wall : 362.827 ms/iter
|
| 52 |
+
--> time hidden by overlap : 8.600 ms/iter
|
| 53 |
+
|
| 54 |
+
=== kernels on device 1, by total time ===
|
| 55 |
+
104.142 ms elementwise_kernel
|
| 56 |
+
73.412 ms device_kernel
|
| 57 |
+
60.052 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 58 |
+
48.238 ms multi_tensor_apply_kernel
|
| 59 |
+
47.684 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 60 |
+
43.994 ms cast_mxfp8_2D_kernel
|
| 61 |
+
41.995 ms vectorized_elementwise_kernel
|
| 62 |
+
24.387 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 63 |
+
23.745 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 64 |
+
16.678 ms unrolled_elementwise_kernel
|
| 65 |
+
11.859 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 66 |
+
REDUCE 11.637 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 67 |
+
10.707 ms ncclDevKernel_AllGather_RING_LL
|
| 68 |
+
8.219 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 69 |
+
7.883 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 70 |
+
7.670 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 71 |
+
6.984 ms triton_poi_fused__to_copy_cat_0
|
| 72 |
+
REDUCE 6.545 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 73 |
+
5.756 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 74 |
+
5.641 ms setGroupedGemmWgradArguments
|
| 75 |
+
5.491 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 76 |
+
4.983 ms gatherTopK
|
| 77 |
+
4.719 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 78 |
+
4.479 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 79 |
+
4.266 ms reduce_kernel
|
| 80 |
+
4.030 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 81 |
+
3.785 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 82 |
+
3.768 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 83 |
+
3.680 ms rmsnorm_bwd_tuned_kernel
|
| 84 |
+
3.473 ms CatArrayBatchedCopy_alignedK_contig
|
| 85 |
+
3.345 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 86 |
+
3.290 ms _scatter_gather_elementwise_kernel
|
| 87 |
+
3.242 ms multi_tensor_swizzle_row_scaling_kernel
|
| 88 |
+
3.128 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 89 |
+
2.989 ms multi_tensor_swizzle_col_scaling_kernel
|
| 90 |
+
2.949 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 91 |
+
2.626 ms fused_rope_backward_kernel
|
| 92 |
+
2.455 ms fused_rope_forward_kernel
|
| 93 |
+
2.367 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 94 |
+
2.256 ms swizzle_col_scaling_kernel
|
| 95 |
+
|
| 96 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) ===
|
| 97 |
+
reduce_grad kernels : 128 (64/iter)
|
| 98 |
+
reduce_grad sum : 9.091 ms/iter
|
| 99 |
+
reduce_grad busy (union) : 9.091 ms/iter
|
| 100 |
+
... overlapped with compute: 8.477 ms/iter ( 93.2 %)
|
| 101 |
+
... exposed on critical path: 0.614 ms/iter ( 6.8 %)
|
| 102 |
+
other GPU work (union) : 306.324 ms/iter
|
| 103 |
+
traced wall : 407.153 ms/iter
|
| 104 |
+
--> time hidden by overlap : 8.477 ms/iter
|
| 105 |
+
|
| 106 |
+
=== kernels on device 2, by total time ===
|
| 107 |
+
104.106 ms elementwise_kernel
|
| 108 |
+
74.640 ms device_kernel
|
| 109 |
+
59.997 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 110 |
+
49.081 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 111 |
+
48.255 ms multi_tensor_apply_kernel
|
| 112 |
+
43.595 ms cast_mxfp8_2D_kernel
|
| 113 |
+
39.678 ms vectorized_elementwise_kernel
|
| 114 |
+
24.151 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 115 |
+
23.808 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 116 |
+
16.711 ms unrolled_elementwise_kernel
|
| 117 |
+
REDUCE 11.705 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 118 |
+
10.891 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 119 |
+
10.771 ms ncclDevKernel_AllGather_RING_LL
|
| 120 |
+
8.251 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 121 |
+
8.007 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 122 |
+
7.711 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 123 |
+
7.003 ms triton_poi_fused__to_copy_cat_0
|
| 124 |
+
REDUCE 6.458 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 125 |
+
6.252 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 126 |
+
5.604 ms setGroupedGemmWgradArguments
|
| 127 |
+
4.997 ms gatherTopK
|
| 128 |
+
4.771 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 129 |
+
4.523 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 130 |
+
4.222 ms reduce_kernel
|
| 131 |
+
4.128 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 132 |
+
3.973 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 133 |
+
3.895 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 134 |
+
3.780 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 135 |
+
3.705 ms rmsnorm_bwd_tuned_kernel
|
| 136 |
+
3.506 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 137 |
+
3.464 ms CatArrayBatchedCopy_alignedK_contig
|
| 138 |
+
3.387 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 139 |
+
3.282 ms _scatter_gather_elementwise_kernel
|
| 140 |
+
3.222 ms multi_tensor_swizzle_row_scaling_kernel
|
| 141 |
+
2.995 ms multi_tensor_swizzle_col_scaling_kernel
|
| 142 |
+
2.630 ms fused_rope_backward_kernel
|
| 143 |
+
2.461 ms fused_rope_forward_kernel
|
| 144 |
+
2.346 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 145 |
+
2.225 ms swizzle_col_scaling_kernel
|
| 146 |
+
2.015 ms triton_poi_fused__to_copy_mul_silu_0
|
| 147 |
+
|
| 148 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) ===
|
| 149 |
+
reduce_grad kernels : 128 (64/iter)
|
| 150 |
+
reduce_grad sum : 9.082 ms/iter
|
| 151 |
+
reduce_grad busy (union) : 9.082 ms/iter
|
| 152 |
+
... overlapped with compute: 8.467 ms/iter ( 93.2 %)
|
| 153 |
+
... exposed on critical path: 0.615 ms/iter ( 6.8 %)
|
| 154 |
+
other GPU work (union) : 304.705 ms/iter
|
| 155 |
+
traced wall : 362.800 ms/iter
|
| 156 |
+
--> time hidden by overlap : 8.467 ms/iter
|
| 157 |
+
|
| 158 |
+
=== kernels on device 3, by total time ===
|
| 159 |
+
104.219 ms elementwise_kernel
|
| 160 |
+
74.904 ms device_kernel
|
| 161 |
+
60.025 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 162 |
+
49.022 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 163 |
+
48.228 ms multi_tensor_apply_kernel
|
| 164 |
+
43.637 ms cast_mxfp8_2D_kernel
|
| 165 |
+
39.885 ms vectorized_elementwise_kernel
|
| 166 |
+
24.080 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 167 |
+
23.387 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 168 |
+
16.721 ms unrolled_elementwise_kernel
|
| 169 |
+
REDUCE 11.160 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 170 |
+
10.756 ms ncclDevKernel_AllGather_RING_LL
|
| 171 |
+
9.843 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 172 |
+
8.314 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 173 |
+
7.956 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 174 |
+
7.815 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 175 |
+
7.008 ms triton_poi_fused__to_copy_cat_0
|
| 176 |
+
REDUCE 6.628 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 177 |
+
5.811 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 178 |
+
5.673 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 179 |
+
5.654 ms setGroupedGemmWgradArguments
|
| 180 |
+
4.976 ms gatherTopK
|
| 181 |
+
4.804 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 182 |
+
4.538 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 183 |
+
4.208 ms reduce_kernel
|
| 184 |
+
3.995 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 185 |
+
3.972 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 186 |
+
3.805 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 187 |
+
3.651 ms rmsnorm_bwd_tuned_kernel
|
| 188 |
+
3.464 ms CatArrayBatchedCopy_alignedK_contig
|
| 189 |
+
3.426 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 190 |
+
3.278 ms _scatter_gather_elementwise_kernel
|
| 191 |
+
3.240 ms multi_tensor_swizzle_row_scaling_kernel
|
| 192 |
+
2.999 ms multi_tensor_swizzle_col_scaling_kernel
|
| 193 |
+
2.997 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 194 |
+
2.625 ms fused_rope_backward_kernel
|
| 195 |
+
2.459 ms fused_rope_forward_kernel
|
| 196 |
+
2.356 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 197 |
+
2.241 ms swizzle_col_scaling_kernel
|
| 198 |
+
2.235 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 199 |
+
|
| 200 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) ===
|
| 201 |
+
reduce_grad kernels : 128 (64/iter)
|
| 202 |
+
reduce_grad sum : 8.894 ms/iter
|
| 203 |
+
reduce_grad busy (union) : 8.894 ms/iter
|
| 204 |
+
... overlapped with compute: 8.288 ms/iter ( 93.2 %)
|
| 205 |
+
... exposed on critical path: 0.606 ms/iter ( 6.8 %)
|
| 206 |
+
other GPU work (union) : 305.557 ms/iter
|
| 207 |
+
traced wall : 362.873 ms/iter
|
| 208 |
+
--> time hidden by overlap : 8.288 ms/iter
|
| 209 |
+
|
| 210 |
+
=== kernels on device 4, by total time ===
|
| 211 |
+
104.178 ms elementwise_kernel
|
| 212 |
+
74.395 ms device_kernel
|
| 213 |
+
59.972 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 214 |
+
48.653 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 215 |
+
48.248 ms multi_tensor_apply_kernel
|
| 216 |
+
43.654 ms cast_mxfp8_2D_kernel
|
| 217 |
+
39.865 ms vectorized_elementwise_kernel
|
| 218 |
+
23.922 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 219 |
+
23.872 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 220 |
+
16.611 ms unrolled_elementwise_kernel
|
| 221 |
+
REDUCE 11.630 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 222 |
+
10.538 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 223 |
+
8.898 ms ncclDevKernel_AllGather_RING_LL
|
| 224 |
+
8.274 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 225 |
+
7.923 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 226 |
+
7.764 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 227 |
+
7.000 ms triton_poi_fused__to_copy_cat_0
|
| 228 |
+
REDUCE 6.429 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 229 |
+
6.110 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 230 |
+
5.481 ms setGroupedGemmWgradArguments
|
| 231 |
+
4.981 ms gatherTopK
|
| 232 |
+
4.802 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 233 |
+
4.536 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 234 |
+
4.236 ms reduce_kernel
|
| 235 |
+
4.017 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 236 |
+
3.852 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 237 |
+
3.794 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 238 |
+
3.672 ms rmsnorm_bwd_tuned_kernel
|
| 239 |
+
3.476 ms CatArrayBatchedCopy_alignedK_contig
|
| 240 |
+
3.377 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 241 |
+
3.285 ms _scatter_gather_elementwise_kernel
|
| 242 |
+
3.240 ms multi_tensor_swizzle_row_scaling_kernel
|
| 243 |
+
3.182 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 244 |
+
3.007 ms multi_tensor_swizzle_col_scaling_kernel
|
| 245 |
+
2.631 ms fused_rope_backward_kernel
|
| 246 |
+
2.501 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 247 |
+
2.464 ms fused_rope_forward_kernel
|
| 248 |
+
2.347 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 249 |
+
2.214 ms swizzle_col_scaling_kernel
|
| 250 |
+
2.013 ms triton_poi_fused__to_copy_mul_silu_0
|
| 251 |
+
|
| 252 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) ===
|
| 253 |
+
reduce_grad kernels : 128 (64/iter)
|
| 254 |
+
reduce_grad sum : 9.029 ms/iter
|
| 255 |
+
reduce_grad busy (union) : 9.029 ms/iter
|
| 256 |
+
... overlapped with compute: 8.401 ms/iter ( 93.0 %)
|
| 257 |
+
... exposed on critical path: 0.628 ms/iter ( 7.0 %)
|
| 258 |
+
other GPU work (union) : 302.106 ms/iter
|
| 259 |
+
traced wall : 362.882 ms/iter
|
| 260 |
+
--> time hidden by overlap : 8.401 ms/iter
|
| 261 |
+
|
| 262 |
+
=== kernels on device 5, by total time ===
|
| 263 |
+
104.221 ms elementwise_kernel
|
| 264 |
+
73.951 ms device_kernel
|
| 265 |
+
59.986 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 266 |
+
48.217 ms multi_tensor_apply_kernel
|
| 267 |
+
48.136 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 268 |
+
43.626 ms cast_mxfp8_2D_kernel
|
| 269 |
+
39.881 ms vectorized_elementwise_kernel
|
| 270 |
+
23.870 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 271 |
+
23.704 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 272 |
+
16.713 ms unrolled_elementwise_kernel
|
| 273 |
+
11.556 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 274 |
+
REDUCE 11.451 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 275 |
+
8.772 ms ncclDevKernel_AllGather_RING_LL
|
| 276 |
+
8.273 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 277 |
+
7.952 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 278 |
+
7.750 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 279 |
+
7.003 ms triton_poi_fused__to_copy_cat_0
|
| 280 |
+
REDUCE 6.542 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 281 |
+
5.850 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 282 |
+
5.627 ms setGroupedGemmWgradArguments
|
| 283 |
+
5.304 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 284 |
+
4.964 ms gatherTopK
|
| 285 |
+
4.773 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 286 |
+
4.531 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 287 |
+
4.258 ms reduce_kernel
|
| 288 |
+
3.994 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 289 |
+
3.897 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 290 |
+
3.781 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 291 |
+
3.680 ms rmsnorm_bwd_tuned_kernel
|
| 292 |
+
3.466 ms CatArrayBatchedCopy_alignedK_contig
|
| 293 |
+
3.397 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 294 |
+
3.301 ms _scatter_gather_elementwise_kernel
|
| 295 |
+
3.198 ms multi_tensor_swizzle_row_scaling_kernel
|
| 296 |
+
2.998 ms multi_tensor_swizzle_col_scaling_kernel
|
| 297 |
+
2.835 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 298 |
+
2.776 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 299 |
+
2.627 ms fused_rope_backward_kernel
|
| 300 |
+
2.460 ms fused_rope_forward_kernel
|
| 301 |
+
2.353 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 302 |
+
2.224 ms swizzle_col_scaling_kernel
|
| 303 |
+
|
| 304 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) ===
|
| 305 |
+
reduce_grad kernels : 128 (64/iter)
|
| 306 |
+
reduce_grad sum : 8.997 ms/iter
|
| 307 |
+
reduce_grad busy (union) : 8.997 ms/iter
|
| 308 |
+
... overlapped with compute: 8.369 ms/iter ( 93.0 %)
|
| 309 |
+
... exposed on critical path: 0.628 ms/iter ( 7.0 %)
|
| 310 |
+
other GPU work (union) : 304.289 ms/iter
|
| 311 |
+
traced wall : 362.570 ms/iter
|
| 312 |
+
--> time hidden by overlap : 8.369 ms/iter
|
| 313 |
+
|
| 314 |
+
=== kernels on device 6, by total time ===
|
| 315 |
+
104.108 ms elementwise_kernel
|
| 316 |
+
74.193 ms device_kernel
|
| 317 |
+
60.031 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 318 |
+
48.600 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 319 |
+
48.179 ms multi_tensor_apply_kernel
|
| 320 |
+
44.105 ms cast_mxfp8_2D_kernel
|
| 321 |
+
39.947 ms vectorized_elementwise_kernel
|
| 322 |
+
24.076 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 323 |
+
23.692 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 324 |
+
16.692 ms unrolled_elementwise_kernel
|
| 325 |
+
REDUCE 11.704 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 326 |
+
10.582 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 327 |
+
8.933 ms ncclDevKernel_AllGather_RING_LL
|
| 328 |
+
8.332 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 329 |
+
8.087 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 330 |
+
7.839 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 331 |
+
7.009 ms triton_poi_fused__to_copy_cat_0
|
| 332 |
+
REDUCE 6.400 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 333 |
+
5.703 ms setGroupedGemmWgradArguments
|
| 334 |
+
5.653 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 335 |
+
5.177 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 336 |
+
4.989 ms gatherTopK
|
| 337 |
+
4.802 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 338 |
+
4.542 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 339 |
+
4.274 ms reduce_kernel
|
| 340 |
+
4.031 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 341 |
+
4.012 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 342 |
+
3.846 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 343 |
+
3.693 ms rmsnorm_bwd_tuned_kernel
|
| 344 |
+
3.469 ms CatArrayBatchedCopy_alignedK_contig
|
| 345 |
+
3.412 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 346 |
+
3.284 ms _scatter_gather_elementwise_kernel
|
| 347 |
+
3.216 ms multi_tensor_swizzle_row_scaling_kernel
|
| 348 |
+
3.005 ms multi_tensor_swizzle_col_scaling_kernel
|
| 349 |
+
2.938 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 350 |
+
2.631 ms fused_rope_backward_kernel
|
| 351 |
+
2.480 ms fused_rope_forward_kernel
|
| 352 |
+
2.365 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 353 |
+
2.252 ms swizzle_col_scaling_kernel
|
| 354 |
+
2.112 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 355 |
+
|
| 356 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) ===
|
| 357 |
+
reduce_grad kernels : 128 (64/iter)
|
| 358 |
+
reduce_grad sum : 9.052 ms/iter
|
| 359 |
+
reduce_grad busy (union) : 9.052 ms/iter
|
| 360 |
+
... overlapped with compute: 8.448 ms/iter ( 93.3 %)
|
| 361 |
+
... exposed on critical path: 0.604 ms/iter ( 6.7 %)
|
| 362 |
+
other GPU work (union) : 304.397 ms/iter
|
| 363 |
+
traced wall : 362.648 ms/iter
|
| 364 |
+
--> time hidden by overlap : 8.448 ms/iter
|
| 365 |
+
|
| 366 |
+
=== kernels on device 7, by total time ===
|
| 367 |
+
104.128 ms elementwise_kernel
|
| 368 |
+
73.310 ms device_kernel
|
| 369 |
+
59.971 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2949124096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
|
| 370 |
+
48.187 ms multi_tensor_apply_kernel
|
| 371 |
+
47.868 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2949124096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
|
| 372 |
+
44.117 ms cast_mxfp8_2D_kernel
|
| 373 |
+
39.906 ms vectorized_elementwise_kernel
|
| 374 |
+
23.914 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
|
| 375 |
+
23.718 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
|
| 376 |
+
16.678 ms unrolled_elementwise_kernel
|
| 377 |
+
12.742 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
|
| 378 |
+
REDUCE 11.820 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
|
| 379 |
+
11.113 ms ncclDevKernel_AllGather_RING_LL
|
| 380 |
+
8.232 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
|
| 381 |
+
7.915 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
|
| 382 |
+
7.784 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
|
| 383 |
+
6.986 ms triton_poi_fused__to_copy_cat_0
|
| 384 |
+
REDUCE 6.644 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o524288896i64i64_tensorptrf32gmemalign16o524288896i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_TVL_0
|
| 385 |
+
6.334 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
|
| 386 |
+
5.636 ms setGroupedGemmWgradArguments
|
| 387 |
+
5.243 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0
|
| 388 |
+
4.973 ms gatherTopK
|
| 389 |
+
4.789 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 390 |
+
4.499 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
|
| 391 |
+
4.266 ms reduce_kernel
|
| 392 |
+
4.024 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_1281_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_al_0
|
| 393 |
+
3.842 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
|
| 394 |
+
3.816 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT
|
| 395 |
+
3.724 ms rmsnorm_bwd_tuned_kernel
|
| 396 |
+
3.469 ms CatArrayBatchedCopy_alignedK_contig
|
| 397 |
+
3.371 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
|
| 398 |
+
3.298 ms _scatter_gather_elementwise_kernel
|
| 399 |
+
3.263 ms multi_tensor_swizzle_row_scaling_kernel
|
| 400 |
+
3.181 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o52428889611i640_CopyAtom_ThrID10_TVLayoutSrc1_0
|
| 401 |
+
2.995 ms multi_tensor_swizzle_col_scaling_kernel
|
| 402 |
+
2.628 ms fused_rope_backward_kernel
|
| 403 |
+
2.450 ms fused_rope_forward_kernel
|
| 404 |
+
2.364 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o368324096i64i64_tensorptri32gmemalign16o1104961_tensorptri32gmemalign16o368321_tensor_0
|
| 405 |
+
2.362 ms triton_red_fused__to_copy_mul_silu_sum_1
|
| 406 |
+
2.252 ms swizzle_col_scaling_kernel
|
| 407 |
+
|
| 408 |
+
=== G4_K8__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) ===
|
| 409 |
+
reduce_grad kernels : 128 (64/iter)
|
| 410 |
+
reduce_grad sum : 9.232 ms/iter
|
| 411 |
+
reduce_grad busy (union) : 9.232 ms/iter
|
| 412 |
+
... overlapped with compute: 8.620 ms/iter ( 93.4 %)
|
| 413 |
+
... exposed on critical path: 0.612 ms/iter ( 6.6 %)
|
| 414 |
+
other GPU work (union) : 306.052 ms/iter
|
| 415 |
+
traced wall : 362.662 ms/iter
|
| 416 |
+
--> time hidden by overlap : 8.620 ms/iter
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
devices=8 reduce=9.074 ms/iter overlap=8.459 ms/iter (93.2%) exposed=0.615 ms/iter status=PASS
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
report : /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep
|
| 2 |
+
kernels : 157909
|
| 3 |
+
devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
|
| 4 |
+
processes: 1
|
| 5 |
+
markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
|
| 6 |
+
PASS: all ranks present and backend markers emitted
|
granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv
CHANGED
|
@@ -5,3 +5,6 @@ G3_K8 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/ti
|
|
| 5 |
G4_K2 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 6 |
G4_K4 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 7 |
G4_K6 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
G4_K2 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 6 |
G4_K4 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K4__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K4__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 7 |
G4_K6 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K6__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K6__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 8 |
+
G4_K8 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K8__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K8__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 9 |
+
G4_K10 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K10__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K10__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|
| 10 |
+
G4_K16 moonep none OK /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/G4_K16__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/granularity_next_b200/tiera_topk_trace/moonep/tierbtrace/tierbtrace_gbs16/G4_K16__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json
|