AbhishekGhosh1998 commited on
Commit
9a2359c
·
verified ·
1 Parent(s): 99db931

granularity results tierb-fixed-moonep-full/moonep/tierbtrace @ 2026-08-30T02:50:00Z from sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0

Browse files
.gitattributes CHANGED
@@ -66,3 +66,4 @@ tierb-fixed-moonep-full/ultraep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__ultraep__m
66
  tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/nsys/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
67
  tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
68
  tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
 
 
66
  tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/nsys/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
67
  tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
68
  tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
69
+ tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf9964e0453e497d76c08d8f285d046dc10a49a585c88e71e7a4c1e46d0d9d7d
3
+ size 5824677
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/moonep_async_overlap.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "devices": 8,
3
+ "reduce_busy_ms_per_iteration": 5.654125,
4
+ "overlapped_ms_per_iteration": 5.428875,
5
+ "exposed_ms_per_iteration": 0.225125,
6
+ "overlap_pct": 96.01618287533438,
7
+ "minimum_overlap_pct": 0.1,
8
+ "status": "PASS"
9
+ }
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/moonep_async_overlap.txt ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ === kernels on device 0, by total time ===
3
+ 43.488 ms device_kernel
4
+ 41.759 ms multi_tensor_apply_kernel
5
+ 40.659 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
6
+ 29.947 ms elementwise_kernel
7
+ 28.850 ms vectorized_elementwise_kernel
8
+ 24.950 ms cast_mxfp8_2D_kernel
9
+ 24.932 ms ncclDevKernel_AllGather_RING_LL
10
+ 15.149 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
11
+ 14.862 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
12
+ 13.157 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
13
+ 8.971 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
14
+ 8.888 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
15
+ 8.660 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
16
+ 7.547 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
17
+ REDUCE 7.402 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
18
+ 6.964 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
19
+ 6.883 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
20
+ 6.099 ms unrolled_elementwise_kernel
21
+ 5.699 ms triton_poi_fused__to_copy_cat_0
22
+ 4.962 ms triton_poi_fused_cat_0
23
+ 3.958 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
24
+ REDUCE 3.769 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
25
+ 3.501 ms setGroupedGemmWgradArguments
26
+ 3.205 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
27
+ 3.194 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
28
+ 2.911 ms rmsnorm_bwd_general_kernel
29
+ 2.218 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
30
+ 2.110 ms triton_red_fused__to_copy_mul_silu_sum_1
31
+ 1.810 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
32
+ 1.761 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
33
+ 1.701 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
34
+ 1.564 ms triton_poi_fused__to_copy_mul_silu_0
35
+ 1.528 ms triton_poi_fused_mul_silu_0
36
+ 1.525 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
37
+ 1.475 ms swizzle_col_scaling_kernel
38
+ 1.407 ms reduce_kernel
39
+ 1.312 ms rmsnorm_fwd_general_kernel
40
+ 1.178 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
41
+ 1.088 ms multi_tensor_swizzle_row_scaling_kernel
42
+ 1.076 ms _scatter_gather_elementwise_kernel
43
+
44
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 0, 2 iteration(s)) ===
45
+ reduce_grad kernels : 40 (20/iter)
46
+ reduce_grad sum : 5.586 ms/iter
47
+ reduce_grad busy (union) : 5.586 ms/iter
48
+ ... overlapped with compute: 5.370 ms/iter ( 96.1 %)
49
+ ... exposed on critical path: 0.216 ms/iter ( 3.9 %)
50
+ other GPU work (union) : 169.985 ms/iter
51
+ traced wall : 188.270 ms/iter
52
+ --> time hidden by overlap : 5.370 ms/iter
53
+
54
+ === kernels on device 1, by total time ===
55
+ 43.299 ms device_kernel
56
+ 41.917 ms multi_tensor_apply_kernel
57
+ 40.230 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
58
+ 30.588 ms elementwise_kernel
59
+ 28.878 ms vectorized_elementwise_kernel
60
+ 24.874 ms cast_mxfp8_2D_kernel
61
+ 23.761 ms ncclDevKernel_AllGather_RING_LL
62
+ 15.118 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
63
+ 15.038 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
64
+ 13.366 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
65
+ 8.768 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
66
+ 8.489 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
67
+ 7.563 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
68
+ REDUCE 7.395 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
69
+ 7.139 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
70
+ 6.971 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
71
+ 6.811 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
72
+ 6.149 ms unrolled_elementwise_kernel
73
+ 5.673 ms triton_poi_fused__to_copy_cat_0
74
+ 4.930 ms triton_poi_fused_cat_0
75
+ 4.109 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
76
+ 4.037 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
77
+ REDUCE 3.982 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
78
+ 3.930 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
79
+ 3.480 ms setGroupedGemmWgradArguments
80
+ 3.029 ms rmsnorm_bwd_general_kernel
81
+ 2.223 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
82
+ 2.120 ms triton_red_fused__to_copy_mul_silu_sum_1
83
+ 2.017 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
84
+ 1.799 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
85
+ 1.715 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
86
+ 1.576 ms triton_poi_fused__to_copy_mul_silu_0
87
+ 1.538 ms triton_poi_fused_mul_silu_0
88
+ 1.512 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
89
+ 1.451 ms swizzle_col_scaling_kernel
90
+ 1.394 ms reduce_kernel
91
+ 1.310 ms rmsnorm_fwd_general_kernel
92
+ 1.220 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
93
+ 1.108 ms multi_tensor_swizzle_row_scaling_kernel
94
+ 1.065 ms _scatter_gather_elementwise_kernel
95
+
96
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 1, 2 iteration(s)) ===
97
+ reduce_grad kernels : 40 (20/iter)
98
+ reduce_grad sum : 5.688 ms/iter
99
+ reduce_grad busy (union) : 5.688 ms/iter
100
+ ... overlapped with compute: 5.451 ms/iter ( 95.8 %)
101
+ ... exposed on critical path: 0.238 ms/iter ( 4.2 %)
102
+ other GPU work (union) : 170.206 ms/iter
103
+ traced wall : 188.135 ms/iter
104
+ --> time hidden by overlap : 5.451 ms/iter
105
+
106
+ === kernels on device 2, by total time ===
107
+ 43.195 ms device_kernel
108
+ 41.886 ms multi_tensor_apply_kernel
109
+ 41.672 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
110
+ 30.465 ms elementwise_kernel
111
+ 28.768 ms vectorized_elementwise_kernel
112
+ 24.836 ms cast_mxfp8_2D_kernel
113
+ 24.210 ms ncclDevKernel_AllGather_RING_LL
114
+ 15.126 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
115
+ 14.745 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
116
+ 13.450 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
117
+ 8.838 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
118
+ 8.504 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
119
+ 7.798 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
120
+ 7.619 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
121
+ REDUCE 7.207 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
122
+ 6.936 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
123
+ 6.841 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
124
+ 6.202 ms unrolled_elementwise_kernel
125
+ 5.663 ms triton_poi_fused__to_copy_cat_0
126
+ 5.615 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
127
+ 4.893 ms triton_poi_fused_cat_0
128
+ REDUCE 4.086 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
129
+ 4.057 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
130
+ 3.880 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
131
+ 3.474 ms setGroupedGemmWgradArguments
132
+ 3.058 ms rmsnorm_bwd_general_kernel
133
+ 2.238 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
134
+ 2.121 ms triton_red_fused__to_copy_mul_silu_sum_1
135
+ 1.913 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
136
+ 1.821 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
137
+ 1.661 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
138
+ 1.572 ms triton_poi_fused__to_copy_mul_silu_0
139
+ 1.569 ms triton_poi_fused_mul_silu_0
140
+ 1.500 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
141
+ 1.454 ms swizzle_col_scaling_kernel
142
+ 1.413 ms reduce_kernel
143
+ 1.295 ms rmsnorm_fwd_general_kernel
144
+ 1.210 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
145
+ 1.099 ms multi_tensor_swizzle_row_scaling_kernel
146
+ 1.076 ms _scatter_gather_elementwise_kernel
147
+
148
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 2, 2 iteration(s)) ===
149
+ reduce_grad kernels : 40 (20/iter)
150
+ reduce_grad sum : 5.647 ms/iter
151
+ reduce_grad busy (union) : 5.647 ms/iter
152
+ ... overlapped with compute: 5.429 ms/iter ( 96.2 %)
153
+ ... exposed on critical path: 0.217 ms/iter ( 3.8 %)
154
+ other GPU work (union) : 171.338 ms/iter
155
+ traced wall : 188.244 ms/iter
156
+ --> time hidden by overlap : 5.429 ms/iter
157
+
158
+ === kernels on device 3, by total time ===
159
+ 44.105 ms device_kernel
160
+ 41.994 ms multi_tensor_apply_kernel
161
+ 39.302 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
162
+ 30.685 ms elementwise_kernel
163
+ 29.089 ms vectorized_elementwise_kernel
164
+ 24.919 ms cast_mxfp8_2D_kernel
165
+ 23.562 ms ncclDevKernel_AllGather_RING_LL
166
+ 15.200 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
167
+ 15.124 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
168
+ 13.189 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
169
+ 8.847 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
170
+ 8.585 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
171
+ 7.630 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
172
+ REDUCE 7.401 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
173
+ 7.023 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
174
+ 6.847 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
175
+ 6.231 ms unrolled_elementwise_kernel
176
+ 5.685 ms triton_poi_fused__to_copy_cat_0
177
+ 5.420 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
178
+ 4.965 ms triton_poi_fused_cat_0
179
+ 4.917 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
180
+ 4.377 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
181
+ 3.995 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
182
+ REDUCE 3.895 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
183
+ 3.379 ms setGroupedGemmWgradArguments
184
+ 3.051 ms rmsnorm_bwd_general_kernel
185
+ 2.243 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
186
+ 2.113 ms triton_red_fused__to_copy_mul_silu_sum_1
187
+ 2.011 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
188
+ 1.826 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
189
+ 1.743 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
190
+ 1.579 ms triton_poi_fused__to_copy_mul_silu_0
191
+ 1.536 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
192
+ 1.524 ms triton_poi_fused_mul_silu_0
193
+ 1.475 ms swizzle_col_scaling_kernel
194
+ 1.414 ms reduce_kernel
195
+ 1.314 ms rmsnorm_fwd_general_kernel
196
+ 1.207 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
197
+ 1.139 ms multi_tensor_swizzle_row_scaling_kernel
198
+ 1.078 ms _scatter_gather_elementwise_kernel
199
+
200
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 3, 2 iteration(s)) ===
201
+ reduce_grad kernels : 40 (20/iter)
202
+ reduce_grad sum : 5.648 ms/iter
203
+ reduce_grad busy (union) : 5.648 ms/iter
204
+ ... overlapped with compute: 5.424 ms/iter ( 96.0 %)
205
+ ... exposed on critical path: 0.224 ms/iter ( 4.0 %)
206
+ other GPU work (union) : 171.016 ms/iter
207
+ traced wall : 188.252 ms/iter
208
+ --> time hidden by overlap : 5.424 ms/iter
209
+
210
+ === kernels on device 4, by total time ===
211
+ 44.301 ms device_kernel
212
+ 41.787 ms multi_tensor_apply_kernel
213
+ 39.259 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
214
+ 30.678 ms elementwise_kernel
215
+ 28.893 ms vectorized_elementwise_kernel
216
+ 24.818 ms cast_mxfp8_2D_kernel
217
+ 23.267 ms ncclDevKernel_AllGather_RING_LL
218
+ 15.239 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
219
+ 15.101 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
220
+ 13.362 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
221
+ 8.906 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
222
+ 8.530 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
223
+ 7.635 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
224
+ REDUCE 7.536 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
225
+ 7.064 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
226
+ 6.862 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
227
+ 6.193 ms unrolled_elementwise_kernel
228
+ 5.690 ms triton_poi_fused__to_copy_cat_0
229
+ 5.678 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
230
+ 4.958 ms triton_poi_fused_cat_0
231
+ 4.700 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
232
+ 4.565 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
233
+ 3.995 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
234
+ REDUCE 3.871 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
235
+ 3.360 ms setGroupedGemmWgradArguments
236
+ 3.038 ms rmsnorm_bwd_general_kernel
237
+ 2.234 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
238
+ 2.119 ms triton_red_fused__to_copy_mul_silu_sum_1
239
+ 2.068 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
240
+ 1.817 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
241
+ 1.736 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
242
+ 1.582 ms triton_poi_fused__to_copy_mul_silu_0
243
+ 1.543 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
244
+ 1.497 ms triton_poi_fused_mul_silu_0
245
+ 1.475 ms swizzle_col_scaling_kernel
246
+ 1.418 ms reduce_kernel
247
+ 1.311 ms rmsnorm_fwd_general_kernel
248
+ 1.209 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
249
+ 1.122 ms multi_tensor_swizzle_row_scaling_kernel
250
+ 1.079 ms _scatter_gather_elementwise_kernel
251
+
252
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 4, 2 iteration(s)) ===
253
+ reduce_grad kernels : 40 (20/iter)
254
+ reduce_grad sum : 5.704 ms/iter
255
+ reduce_grad busy (union) : 5.704 ms/iter
256
+ ... overlapped with compute: 5.484 ms/iter ( 96.2 %)
257
+ ... exposed on critical path: 0.219 ms/iter ( 3.8 %)
258
+ other GPU work (union) : 171.090 ms/iter
259
+ traced wall : 188.344 ms/iter
260
+ --> time hidden by overlap : 5.484 ms/iter
261
+
262
+ === kernels on device 5, by total time ===
263
+ 43.196 ms device_kernel
264
+ 41.914 ms multi_tensor_apply_kernel
265
+ 41.601 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
266
+ 30.508 ms elementwise_kernel
267
+ 28.676 ms vectorized_elementwise_kernel
268
+ 26.302 ms ncclDevKernel_AllGather_RING_LL
269
+ 25.015 ms cast_mxfp8_2D_kernel
270
+ 15.130 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
271
+ 14.913 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
272
+ 13.165 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
273
+ 9.320 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
274
+ 8.864 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
275
+ 8.713 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
276
+ 7.551 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
277
+ REDUCE 7.255 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
278
+ 6.934 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
279
+ 6.813 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
280
+ 6.141 ms unrolled_elementwise_kernel
281
+ 5.664 ms triton_poi_fused__to_copy_cat_0
282
+ 5.632 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
283
+ 4.904 ms triton_poi_fused_cat_0
284
+ 3.904 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
285
+ REDUCE 3.861 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
286
+ 3.444 ms setGroupedGemmWgradArguments
287
+ 3.023 ms rmsnorm_bwd_general_kernel
288
+ 2.682 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
289
+ 2.237 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
290
+ 2.118 ms triton_red_fused__to_copy_mul_silu_sum_1
291
+ 1.813 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
292
+ 1.677 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
293
+ 1.576 ms triton_poi_fused__to_copy_mul_silu_0
294
+ 1.555 ms triton_poi_fused_mul_silu_0
295
+ 1.509 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
296
+ 1.445 ms swizzle_col_scaling_kernel
297
+ 1.396 ms reduce_kernel
298
+ 1.307 ms rmsnorm_fwd_general_kernel
299
+ 1.286 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
300
+ 1.219 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
301
+ 1.082 ms multi_tensor_swizzle_row_scaling_kernel
302
+ 1.065 ms _scatter_gather_elementwise_kernel
303
+
304
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 5, 2 iteration(s)) ===
305
+ reduce_grad kernels : 40 (20/iter)
306
+ reduce_grad sum : 5.558 ms/iter
307
+ reduce_grad busy (union) : 5.558 ms/iter
308
+ ... overlapped with compute: 5.326 ms/iter ( 95.8 %)
309
+ ... exposed on critical path: 0.232 ms/iter ( 4.2 %)
310
+ other GPU work (union) : 171.203 ms/iter
311
+ traced wall : 188.270 ms/iter
312
+ --> time hidden by overlap : 5.326 ms/iter
313
+
314
+ === kernels on device 6, by total time ===
315
+ 43.311 ms device_kernel
316
+ 41.861 ms multi_tensor_apply_kernel
317
+ 40.487 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
318
+ 30.548 ms elementwise_kernel
319
+ 28.861 ms vectorized_elementwise_kernel
320
+ 24.975 ms cast_mxfp8_2D_kernel
321
+ 24.815 ms ncclDevKernel_AllGather_RING_LL
322
+ 15.104 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
323
+ 14.875 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
324
+ 13.106 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
325
+ 8.831 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
326
+ 8.633 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
327
+ 8.282 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
328
+ 7.567 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
329
+ REDUCE 7.415 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
330
+ 7.025 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
331
+ 6.843 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
332
+ 6.189 ms unrolled_elementwise_kernel
333
+ 5.681 ms triton_poi_fused__to_copy_cat_0
334
+ 4.923 ms triton_poi_fused_cat_0
335
+ REDUCE 4.024 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
336
+ 3.955 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
337
+ 3.448 ms setGroupedGemmWgradArguments
338
+ 3.072 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
339
+ 3.067 ms rmsnorm_bwd_general_kernel
340
+ 2.215 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
341
+ 2.117 ms triton_red_fused__to_copy_mul_silu_sum_1
342
+ 1.802 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
343
+ 1.739 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
344
+ 1.705 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
345
+ 1.580 ms triton_poi_fused__to_copy_mul_silu_0
346
+ 1.563 ms triton_poi_fused_mul_silu_0
347
+ 1.518 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
348
+ 1.470 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
349
+ 1.457 ms swizzle_col_scaling_kernel
350
+ 1.397 ms reduce_kernel
351
+ 1.312 ms rmsnorm_fwd_general_kernel
352
+ 1.220 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
353
+ 1.078 ms _scatter_gather_elementwise_kernel
354
+ 1.077 ms multi_tensor_swizzle_row_scaling_kernel
355
+
356
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 6, 2 iteration(s)) ===
357
+ reduce_grad kernels : 40 (20/iter)
358
+ reduce_grad sum : 5.719 ms/iter
359
+ reduce_grad busy (union) : 5.719 ms/iter
360
+ ... overlapped with compute: 5.491 ms/iter ( 96.0 %)
361
+ ... exposed on critical path: 0.228 ms/iter ( 4.0 %)
362
+ other GPU work (union) : 168.985 ms/iter
363
+ traced wall : 188.376 ms/iter
364
+ --> time hidden by overlap : 5.491 ms/iter
365
+
366
+ === kernels on device 7, by total time ===
367
+ 43.569 ms device_kernel
368
+ 41.940 ms multi_tensor_apply_kernel
369
+ 40.673 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL
370
+ 30.640 ms elementwise_kernel
371
+ 28.767 ms vectorized_elementwise_kernel
372
+ 24.843 ms cast_mxfp8_2D_kernel
373
+ 23.460 ms ncclDevKernel_AllGather_RING_LL
374
+ 15.104 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40963584i64i64_tensorptrbf16gmemalign16o2785283584i64i64_tensorptri32gmemalign16o327681_tensorptri32_0
375
+ 15.014 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT
376
+ 13.108 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2785283584i64i64_tensorptrbf16gmemalign16o40963584i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0
377
+ 8.765 ms nvjet_qqsss_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NTT
378
+ 8.455 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_TNT
379
+ 7.578 ms nvjet_qqtst_256x128_128x5_2x4_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT
380
+ REDUCE 7.547 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276803584i64i64_tensorptrf32gmemalign16o3276803584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
381
+ 6.921 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT
382
+ 6.828 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0
383
+ 6.727 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0
384
+ 6.176 ms unrolled_elementwise_kernel
385
+ 5.687 ms triton_poi_fused__to_copy_cat_0
386
+ 5.324 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL
387
+ 4.933 ms triton_poi_fused_cat_0
388
+ 4.305 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327680358411i640_CopyAtom_ThrID10_TVLayoutSrc_0
389
+ 3.869 ms nvjet_qqtst_128x256_128x6_2x1_2cta_v_bz_Avec32UE8M0_Bvec32UE8M0_NNT
390
+ REDUCE 3.818 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293762560i64i64_tensorptrf32gmemalign16o2293762560i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0
391
+ 3.345 ms setGroupedGemmWgradArguments
392
+ 3.058 ms rmsnorm_bwd_general_kernel
393
+ 2.214 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0
394
+ 2.120 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376256011i640_CopyAtom_ThrID10_TVLayoutSrc_0
395
+ 2.117 ms triton_red_fused__to_copy_mul_silu_sum_1
396
+ 1.820 ms nvjet_tst_320x192_64x4_2x2_2cta_h_bz_NNT
397
+ 1.696 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT
398
+ 1.573 ms triton_poi_fused__to_copy_mul_silu_0
399
+ 1.562 ms triton_poi_fused_mul_silu_0
400
+ 1.508 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT
401
+ 1.467 ms swizzle_col_scaling_kernel
402
+ 1.407 ms reduce_kernel
403
+ 1.311 ms rmsnorm_fwd_general_kernel
404
+ 1.209 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0
405
+ 1.104 ms multi_tensor_swizzle_row_scaling_kernel
406
+ 1.084 ms _scatter_gather_elementwise_kernel
407
+
408
+ === QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.sqlite (device 7, 2 iteration(s)) ===
409
+ reduce_grad kernels : 40 (20/iter)
410
+ reduce_grad sum : 5.683 ms/iter
411
+ reduce_grad busy (union) : 5.683 ms/iter
412
+ ... overlapped with compute: 5.456 ms/iter ( 96.0 %)
413
+ ... exposed on critical path: 0.227 ms/iter ( 4.0 %)
414
+ other GPU work (union) : 171.022 ms/iter
415
+ traced wall : 188.320 ms/iter
416
+ --> time hidden by overlap : 5.456 ms/iter
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/moonep_async_overlap_summary.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ devices=8 reduce=5.654 ms/iter overlap=5.429 ms/iter (96.0%) exposed=0.225 ms/iter status=PASS
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/train.log CHANGED
@@ -993,3 +993,84 @@ INFO:moonep.api:MoonEP comm buffer sizes:
993
  [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [9856, 3456, 3072, 4736, 512, 5632, 1536, 896, 3712, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[9, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [13, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1]]
994
  [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [1792, 5120, 4608, 6144, 2560, 2816, 6400, 3712, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1], [34, -1, -1, -1, -1, -1, -1, -1]]
995
  [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [2048, 2176, 768, 8448, 1664, 2048, 9344, 3072, 3712, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[30, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [1, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [56, -1, -1, -1, -1, -1, -1, -1], [37, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1]]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
993
  [MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [9856, 3456, 3072, 4736, 512, 5632, 1536, 896, 3712, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[9, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [13, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1]]
994
  [MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [1792, 5120, 4608, 6144, 2560, 2816, 6400, 3712, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1], [52, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [6, -1, -1, -1, -1, -1, -1, -1], [34, -1, -1, -1, -1, -1, -1, -1]]
995
  [MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [2048, 2176, 768, 8448, 1664, 2048, 9344, 3072, 3712, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[30, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [1, -1, -1, -1, -1, -1, -1, -1], [50, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [56, -1, -1, -1, -1, -1, -1, -1], [37, -1, -1, -1, -1, -1, -1, -1], [9, -1, -1, -1, -1, -1, -1, -1]]
996
+ [2026-08-30 02:48:33] iteration 1/ 10 | consumed samples: 8 | elapsed time per iteration (ms): 20490.0 | throughput per GPU (TFLOP/s/GPU): 6.0 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097928E+01 | loss scale: 1.0 | grad norm: 29.871 | number of skipped iterations: 0 | number of nan iterations: 0 |
997
+ Number of parameters in transformer block in billions: 20.11
998
+ Number of parameters in embedding layers in billions: 0.36
999
+ Total number of parameters in billions: 20.47
1000
+ Number of parameters in most loaded shard in billions: 20.4724
1001
+ Theoretical memory footprints: weight and optimizer=146430.14 MB
1002
+ [Rank 0] (after 1 iterations) memory (MB) | allocated: 60534.46875 | max allocated: 60534.46923828125 | reserved: 62900.0 | max reserved: 62900.0
1003
+ [2026-08-30 02:48:33] iteration 2/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 247.2 | throughput per GPU (TFLOP/s/GPU): 499.6 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097878E+01 | loss scale: 1.0 | grad norm: 30.630 | number of skipped iterations: 0 | number of nan iterations: 0 |
1004
+ [2026-08-30 02:48:33] iteration 3/ 10 | consumed samples: 24 | elapsed time per iteration (ms): 209.9 | throughput per GPU (TFLOP/s/GPU): 588.3 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.537870E+00 | loss scale: 1.0 | grad norm: 32.328 | number of skipped iterations: 0 | number of nan iterations: 0 |
1005
+ Capture CUDA graph for training!!!
1006
+ CUDA graph capture done!!!
1007
+ [2026-08-30 02:48:34] iteration 4/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 783.5 | throughput per GPU (TFLOP/s/GPU): 157.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.108238E+01 | loss scale: 1.0 | grad norm: 106.897 | number of skipped iterations: 0 | number of nan iterations: 0 |
1008
+ [2026-08-30 02:48:34] iteration 5/ 10 | consumed samples: 40 | elapsed time per iteration (ms): 180.8 | throughput per GPU (TFLOP/s/GPU): 682.8 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.997957E+00 | loss scale: 1.0 | grad norm: 26.196 | number of skipped iterations: 0 | number of nan iterations: 0 |
1009
+ [2026-08-30 02:48:34] iteration 6/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 180.7 | throughput per GPU (TFLOP/s/GPU): 683.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.049965E+01 | loss scale: 1.0 | grad norm: 8.726 | number of skipped iterations: 0 | number of nan iterations: 0 |
1010
+ [2026-08-30 02:48:37] iteration 7/ 10 | consumed samples: 56 | elapsed time per iteration (ms): 2917.7 | throughput per GPU (TFLOP/s/GPU): 42.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.442690E+00 | loss scale: 1.0 | grad norm: 5.413 | number of skipped iterations: 0 | number of nan iterations: 0 |
1011
+ [2026-08-30 02:48:37] iteration 8/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 186.2 | throughput per GPU (TFLOP/s/GPU): 663.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.964504E+00 | loss scale: 1.0 | grad norm: 4.556 | number of skipped iterations: 0 | number of nan iterations: 0 |
1012
+ Capture range started in the application.
1013
+ Capture range ended in the application.
1014
+ Generating '/tmp/nsys-report-5085.qdstrm'
1015
+
1016
+
1017
+ [2026-08-30 02:49:18] iteration 9/ 10 | consumed samples: 72 | elapsed time per iteration (ms): 40624.4 | throughput per GPU (TFLOP/s/GPU): 3.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.748680E+00 | loss scale: 1.0 | grad norm: 4.300 | number of skipped iterations: 0 | number of nan iterations: 0 |
1018
+ [2026-08-30 02:49:18] iteration 10/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 185.0 | throughput per GPU (TFLOP/s/GPU): 667.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.688493E+00 | loss scale: 1.0 | grad norm: 4.428 | number of skipped iterations: 0 | number of nan iterations: 0 |
1019
+ [after training is done] datetime: 2026-08-30 02:49:18
1020
+ [rank0]:[W830 02:49:20.407650089 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1021
+ [rank1]:[W830 02:49:21.483277895 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1022
+ [rank5]:[W830 02:49:21.533068668 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1023
+ [rank3]:[W830 02:49:21.533582983 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1024
+ [rank2]:[W830 02:49:21.533642158 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1025
+ [rank6]:[W830 02:49:21.536994791 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1026
+ [rank4]:[W830 02:49:21.829376424 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1027
+ [rank7]:[W830 02:49:22.342994989 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
1028
+ [rank1]:[W830 02:49:24.454973039 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
1029
+ pure virtual method called
1030
+ terminate called without an active exception
1031
+ [rank5]:[W830 02:49:24.487316028 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator())
1032
+ pure virtual method called
1033
+ terminate called without an active exception
1034
+ W0830 02:49:24.782000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429792 closing signal SIGTERM
1035
+ W0830 02:49:24.783000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429794 closing signal SIGTERM
1036
+ W0830 02:49:24.784000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429795 closing signal SIGTERM
1037
+ W0830 02:49:24.785000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429796 closing signal SIGTERM
1038
+ W0830 02:49:24.785000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429797 closing signal SIGTERM
1039
+ W0830 02:49:24.786000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429798 closing signal SIGTERM
1040
+ W0830 02:49:24.786000 429583 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 429799 closing signal SIGTERM
1041
+ E0830 02:49:26.755000 429583 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 1 (pid: 429793) of binary: /usr/bin/python
1042
+ Traceback (most recent call last):
1043
+ File "/usr/local/bin/torchrun", line 8, in <module>
1044
+ sys.exit(main())
1045
+ ^^^^^^
1046
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
1047
+ return f(*args, **kwargs)
1048
+ ^^^^^^^^^^^^^^^^^^
1049
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main
1050
+ run(args)
1051
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run
1052
+ elastic_launch(
1053
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__
1054
+ return launch_agent(self._config, self._entrypoint, list(args))
1055
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1056
+ File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
1057
+ raise ChildFailedError(
1058
+ torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
1059
+ ============================================================
1060
+ pretrain_gpt.py FAILED
1061
+ ------------------------------------------------------------
1062
+ Failures:
1063
+ <NO_OTHER_FAILURES>
1064
+ ------------------------------------------------------------
1065
+ Root Cause (first observed failure):
1066
+ [0]:
1067
+ time : 2026-08-30_02:49:24
1068
+ host : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0
1069
+ rank : 1 (local_rank: 1)
1070
+ exitcode : -6 (pid: 429793)
1071
+ error_file: <N/A>
1072
+ traceback : Signal 6 (SIGABRT) received by PID 429793
1073
+ ============================================================
1074
+ Generated:
1075
+ /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep
1076
+ There are no active sessions.
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/verify_trace_ranks.log ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ report : /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep
2
+ kernels : 58910
3
+ devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7]
4
+ processes: 1
5
+ markers : 5440 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7]
6
+ PASS: all ranks present and backend markers emitted
tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/trace_results.tsv CHANGED
@@ -2,3 +2,4 @@ model backend capacity status report async_overlap_json
2
  QWEN3_235B moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/nsys/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/nsys/moonep_async_overlap.json
3
  QWEN3_30B moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/moonep_async_overlap.json
4
  DSV2_LITE_L24 moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/moonep_async_overlap.json
 
 
2
  QWEN3_235B moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/nsys/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/nsys/moonep_async_overlap.json
3
  QWEN3_30B moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/nsys/moonep_async_overlap.json
4
  DSV2_LITE_L24 moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__async/nsys/moonep_async_overlap.json
5
+ QWEN2_57B moonep none OK /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async.nsys-rep /workspace/tierb-fixed-moonep-full/moonep/tierbtrace/tierbtrace_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__async/nsys/moonep_async_overlap.json