diff --git a/.gitattributes b/.gitattributes index 01cf83d94ed592a9914396e7a701712063b24470..85dd8492fbcbb20f8e61a440b765f922f7f591a7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -78,3 +78,6 @@ tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G1__ultraep__matche tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G2__ultraep__matched__seq4096__gbs16__cap1p08203125/nsys/G2__ultraep__matched__seq4096__gbs16__cap1p08203125.nsys-rep filter=lfs diff=lfs merge=lfs -text tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G3__ultraep__matched__seq4096__gbs16__cap1p072265625/nsys/G3__ultraep__matched__seq4096__gbs16__cap1p072265625.nsys-rep filter=lfs diff=lfs merge=lfs -text tiera-rerun-fixed-moonep/ultraep/tierbtrace/tierbtrace_gbs16/G4__ultraep__matched__seq4096__gbs16__cap1p095703125/nsys/G4__ultraep__matched__seq4096__gbs16__cap1p095703125.nsys-rep filter=lfs diff=lfs merge=lfs -text +tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text +tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text +tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep filter=lfs diff=lfs merge=lfs -text diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep new file mode 100644 index 0000000000000000000000000000000000000000..42296590a3fec96c2f31918c1fba328f2cbb47ca --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d454bc930b451579b46b31a3e7de18980bfce3e0e66ed011dab8a879f5052faa +size 9671848 diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json new file mode 100644 index 0000000000000000000000000000000000000000..d57cdeda10e454befecc34807d2819d722d335d2 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json @@ -0,0 +1,9 @@ +{ + "devices": 8, + "reduce_busy_ms_per_iteration": 109.24575, + "overlapped_ms_per_iteration": 58.440625, + "exposed_ms_per_iteration": 50.805125, + "overlap_pct": 53.49464395640105, + "minimum_overlap_pct": 0.1, + "status": "PASS" +} diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt new file mode 100644 index 0000000000000000000000000000000000000000..30e9d74141631ef672ce2d841a1263bc0b4eaf8c --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt @@ -0,0 +1,416 @@ + +=== kernels on device 0, by total time === + 232.537 ms device_kernel + REDUCE 155.815 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 109.559 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 62.716 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 52.685 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 50.905 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 47.933 ms multi_tensor_apply_kernel + 36.710 ms cast_mxfp8_2D_kernel + 32.449 ms elementwise_kernel + 30.104 ms vectorized_elementwise_kernel + 28.983 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.574 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.304 ms triton_poi_fused__to_copy_cat_0 + 21.589 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 16.731 ms unrolled_elementwise_kernel + 16.082 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.482 ms ncclDevKernel_AllGather_RING_LL + 10.454 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.708 ms setGroupedGemmWgradArguments + 8.183 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.737 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.632 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.015 ms triton_poi_fused__to_copy_mul_silu_0 + 5.923 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.783 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.431 ms rmsnorm_bwd_tuned_kernel + 3.961 ms CatArrayBatchedCopy_alignedK_contig + 3.938 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.821 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.595 ms reduce_kernel + 3.508 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.414 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.111 ms fused_rope_backward_kernel + 2.845 ms _scatter_gather_elementwise_kernel + 2.606 ms multi_tensor_swizzle_row_scaling_kernel + 2.447 ms fused_rope_forward_kernel + 2.420 ms multi_tensor_swizzle_col_scaling_kernel + 2.267 ms swizzle_col_scaling_kernel + 1.886 ms rmsnorm_fwd_tuned_kernel + 1.790 ms triton_poi_fused_add_0 + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 109.265 ms/iter + reduce_grad busy (union) : 109.265 ms/iter + ... overlapped with compute: 58.042 ms/iter ( 53.1 %) + ... exposed on critical path: 51.223 ms/iter ( 46.9 %) + other GPU work (union) : 425.134 ms/iter + traced wall : 494.975 ms/iter + --> time hidden by overlap : 58.042 ms/iter + +=== kernels on device 1, by total time === + 232.734 ms device_kernel + REDUCE 154.081 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 92.807 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 75.101 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + REDUCE 63.763 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 48.211 ms multi_tensor_apply_kernel + 46.010 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 36.926 ms cast_mxfp8_2D_kernel + 33.138 ms elementwise_kernel + 30.031 ms vectorized_elementwise_kernel + 29.104 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.997 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.084 ms triton_poi_fused__to_copy_cat_0 + 23.511 ms ncclDevKernel_AllGather_RING_LL + 21.537 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 17.051 ms unrolled_elementwise_kernel + 15.337 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.482 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.678 ms setGroupedGemmWgradArguments + 8.186 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.741 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.614 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.079 ms triton_poi_fused__to_copy_mul_silu_0 + 5.944 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.793 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.598 ms rmsnorm_bwd_tuned_kernel + 3.971 ms CatArrayBatchedCopy_alignedK_contig + 3.921 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.781 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.596 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.514 ms reduce_kernel + 3.416 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.143 ms fused_rope_backward_kernel + 2.816 ms _scatter_gather_elementwise_kernel + 2.594 ms multi_tensor_swizzle_row_scaling_kernel + 2.566 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 2.457 ms fused_rope_forward_kernel + 2.450 ms multi_tensor_swizzle_col_scaling_kernel + 2.243 ms swizzle_col_scaling_kernel + 1.893 ms rmsnorm_fwd_tuned_kernel + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 108.922 ms/iter + reduce_grad busy (union) : 108.922 ms/iter + ... overlapped with compute: 58.675 ms/iter ( 53.9 %) + ... exposed on critical path: 50.247 ms/iter ( 46.1 %) + other GPU work (union) : 433.322 ms/iter + traced wall : 501.495 ms/iter + --> time hidden by overlap : 58.675 ms/iter + +=== kernels on device 2, by total time === + 230.190 ms device_kernel + REDUCE 154.378 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 94.558 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 74.068 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + REDUCE 63.532 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 48.174 ms multi_tensor_apply_kernel + 47.151 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 37.000 ms cast_mxfp8_2D_kernel + 33.122 ms elementwise_kernel + 30.169 ms vectorized_elementwise_kernel + 29.298 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.164 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 24.908 ms triton_poi_fused__to_copy_cat_0 + 21.530 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 17.184 ms unrolled_elementwise_kernel + 15.745 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.554 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.606 ms setGroupedGemmWgradArguments + 8.362 ms ncclDevKernel_AllGather_RING_LL + 8.218 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.740 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.696 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.022 ms triton_poi_fused__to_copy_mul_silu_0 + 5.989 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.838 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.547 ms rmsnorm_bwd_tuned_kernel + 3.985 ms CatArrayBatchedCopy_alignedK_contig + 3.848 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.830 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.654 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.535 ms reduce_kernel + 3.416 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.134 ms fused_rope_backward_kernel + 2.862 ms _scatter_gather_elementwise_kernel + 2.622 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 2.588 ms multi_tensor_swizzle_row_scaling_kernel + 2.476 ms fused_rope_forward_kernel + 2.406 ms multi_tensor_swizzle_col_scaling_kernel + 2.264 ms swizzle_col_scaling_kernel + 1.892 ms rmsnorm_fwd_tuned_kernel + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 108.955 ms/iter + reduce_grad busy (union) : 108.955 ms/iter + ... overlapped with compute: 58.603 ms/iter ( 53.8 %) + ... exposed on critical path: 50.352 ms/iter ( 46.2 %) + other GPU work (union) : 425.739 ms/iter + traced wall : 494.216 ms/iter + --> time hidden by overlap : 58.603 ms/iter + +=== kernels on device 3, by total time === + 234.150 ms device_kernel + REDUCE 156.439 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 111.851 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 62.585 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 55.020 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 48.161 ms multi_tensor_apply_kernel + 43.500 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 36.850 ms cast_mxfp8_2D_kernel + 33.223 ms elementwise_kernel + 30.624 ms vectorized_elementwise_kernel + 29.179 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.485 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.197 ms triton_poi_fused__to_copy_cat_0 + 21.574 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 17.082 ms unrolled_elementwise_kernel + 15.473 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.525 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.869 ms ncclDevKernel_AllGather_RING_LL + 8.734 ms setGroupedGemmWgradArguments + 8.173 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.755 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.738 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.067 ms triton_poi_fused__to_copy_mul_silu_0 + 5.965 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.821 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.563 ms rmsnorm_bwd_tuned_kernel + 3.978 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.968 ms CatArrayBatchedCopy_alignedK_contig + 3.837 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.638 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.514 ms reduce_kernel + 3.450 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.130 ms fused_rope_backward_kernel + 2.838 ms _scatter_gather_elementwise_kernel + 2.686 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 2.638 ms multi_tensor_swizzle_row_scaling_kernel + 2.482 ms fused_rope_forward_kernel + 2.454 ms multi_tensor_swizzle_col_scaling_kernel + 2.262 ms swizzle_col_scaling_kernel + 1.893 ms rmsnorm_fwd_tuned_kernel + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 109.512 ms/iter + reduce_grad busy (union) : 109.512 ms/iter + ... overlapped with compute: 58.530 ms/iter ( 53.4 %) + ... exposed on critical path: 50.982 ms/iter ( 46.6 %) + other GPU work (union) : 425.696 ms/iter + traced wall : 531.982 ms/iter + --> time hidden by overlap : 58.530 ms/iter + +=== kernels on device 4, by total time === + 232.303 ms device_kernel + REDUCE 156.283 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 109.634 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 62.579 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 53.627 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 49.043 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 48.214 ms multi_tensor_apply_kernel + 36.810 ms cast_mxfp8_2D_kernel + 33.300 ms elementwise_kernel + 30.213 ms vectorized_elementwise_kernel + 29.279 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.478 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.244 ms triton_poi_fused__to_copy_cat_0 + 21.467 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 17.089 ms unrolled_elementwise_kernel + 15.643 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.591 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.966 ms ncclDevKernel_AllGather_RING_LL + 8.875 ms setGroupedGemmWgradArguments + 8.197 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.742 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.706 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.067 ms triton_poi_fused__to_copy_mul_silu_0 + 5.993 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.843 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.560 ms rmsnorm_bwd_tuned_kernel + 4.020 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.974 ms CatArrayBatchedCopy_alignedK_contig + 3.832 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.618 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.548 ms reduce_kernel + 3.456 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.128 ms fused_rope_backward_kernel + 2.855 ms _scatter_gather_elementwise_kernel + 2.626 ms multi_tensor_swizzle_row_scaling_kernel + 2.457 ms fused_rope_forward_kernel + 2.450 ms multi_tensor_swizzle_col_scaling_kernel + 2.262 ms swizzle_col_scaling_kernel + 1.893 ms rmsnorm_fwd_tuned_kernel + 1.815 ms triton_poi_fused_add_0 + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 109.431 ms/iter + reduce_grad busy (union) : 109.431 ms/iter + ... overlapped with compute: 58.611 ms/iter ( 53.6 %) + ... exposed on critical path: 50.820 ms/iter ( 46.4 %) + other GPU work (union) : 424.447 ms/iter + traced wall : 494.369 ms/iter + --> time hidden by overlap : 58.611 ms/iter + +=== kernels on device 5, by total time === + 229.723 ms device_kernel + REDUCE 153.790 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 87.114 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 85.452 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + REDUCE 63.862 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 48.234 ms multi_tensor_apply_kernel + 44.522 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 37.040 ms cast_mxfp8_2D_kernel + 33.068 ms elementwise_kernel + 29.991 ms vectorized_elementwise_kernel + 29.293 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.474 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 24.920 ms triton_poi_fused__to_copy_cat_0 + 21.505 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 17.069 ms unrolled_elementwise_kernel + 15.453 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.841 ms ncclDevKernel_AllGather_RING_LL + 10.531 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.765 ms setGroupedGemmWgradArguments + 8.263 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.742 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.719 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.046 ms triton_poi_fused__to_copy_mul_silu_0 + 5.984 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.834 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.605 ms rmsnorm_bwd_tuned_kernel + 3.979 ms CatArrayBatchedCopy_alignedK_contig + 3.875 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.816 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.596 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.511 ms reduce_kernel + 3.376 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.144 ms fused_rope_backward_kernel + 2.925 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 2.823 ms _scatter_gather_elementwise_kernel + 2.582 ms multi_tensor_swizzle_row_scaling_kernel + 2.475 ms fused_rope_forward_kernel + 2.424 ms multi_tensor_swizzle_col_scaling_kernel + 2.242 ms swizzle_col_scaling_kernel + 1.899 ms rmsnorm_fwd_tuned_kernel + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 108.826 ms/iter + reduce_grad busy (union) : 108.826 ms/iter + ... overlapped with compute: 58.653 ms/iter ( 53.9 %) + ... exposed on critical path: 50.173 ms/iter ( 46.1 %) + other GPU work (union) : 427.265 ms/iter + traced wall : 495.128 ms/iter + --> time hidden by overlap : 58.653 ms/iter + +=== kernels on device 6, by total time === + 231.663 ms device_kernel + REDUCE 157.279 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 113.340 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 62.173 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 55.110 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 48.219 ms multi_tensor_apply_kernel + 44.531 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 36.828 ms cast_mxfp8_2D_kernel + 33.165 ms elementwise_kernel + 29.799 ms vectorized_elementwise_kernel + 29.089 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.346 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.188 ms triton_poi_fused__to_copy_cat_0 + 21.527 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 16.942 ms unrolled_elementwise_kernel + 15.869 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.454 ms ncclDevKernel_AllGather_RING_LL + 10.268 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.796 ms setGroupedGemmWgradArguments + 8.181 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.739 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.603 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.013 ms triton_poi_fused__to_copy_mul_silu_0 + 5.913 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.780 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.526 ms rmsnorm_bwd_tuned_kernel + 3.944 ms CatArrayBatchedCopy_alignedK_contig + 3.881 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.832 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.598 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.401 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.374 ms reduce_kernel + 3.114 ms fused_rope_backward_kernel + 2.784 ms _scatter_gather_elementwise_kernel + 2.714 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 2.622 ms multi_tensor_swizzle_row_scaling_kernel + 2.453 ms fused_rope_forward_kernel + 2.426 ms multi_tensor_swizzle_col_scaling_kernel + 2.236 ms swizzle_col_scaling_kernel + 1.883 ms rmsnorm_fwd_tuned_kernel + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 109.726 ms/iter + reduce_grad busy (union) : 109.726 ms/iter + ... overlapped with compute: 57.996 ms/iter ( 52.9 %) + ... exposed on critical path: 51.730 ms/iter ( 47.1 %) + other GPU work (union) : 425.576 ms/iter + traced wall : 494.923 ms/iter + --> time hidden by overlap : 57.996 ms/iter + +=== kernels on device 7, by total time === + 229.957 ms device_kernel + REDUCE 155.726 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 105.501 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 62.933 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o3276814336i64i64_tensorptrf32gmemalign16o3276814336i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 58.493 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 52.107 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o327681433611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 48.226 ms multi_tensor_apply_kernel + 36.808 ms cast_mxfp8_2D_kernel + 33.145 ms elementwise_kernel + 30.131 ms vectorized_elementwise_kernel + 29.044 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 27.840 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 24.994 ms triton_poi_fused__to_copy_cat_0 + 21.499 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o675844096i64i64_tensorptri32gmemalign16o81921_tensorptri32gm_0 + 17.114 ms unrolled_elementwise_kernel + 15.468 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o675844096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o81921_tensorptri32gmem_0 + 10.369 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.685 ms setGroupedGemmWgradArguments + 8.319 ms ncclDevKernel_AllGather_RING_LL + 8.162 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.740 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.678 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.989 ms triton_poi_fused__to_copy_mul_silu_0 + 5.918 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.771 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.555 ms rmsnorm_bwd_tuned_kernel + 3.980 ms CatArrayBatchedCopy_alignedK_contig + 3.875 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.841 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.643 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_81_tensorptri32_gmem_align16_o_81921_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_align_0 + 3.561 ms reduce_kernel + 3.374 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.132 ms fused_rope_backward_kernel + 2.874 ms _scatter_gather_elementwise_kernel + 2.596 ms multi_tensor_swizzle_row_scaling_kernel + 2.461 ms fused_rope_forward_kernel + 2.414 ms multi_tensor_swizzle_col_scaling_kernel + 2.413 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 2.273 ms swizzle_col_scaling_kernel + 1.897 ms rmsnorm_fwd_tuned_kernel + +=== G1__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 109.329 ms/iter + reduce_grad busy (union) : 109.329 ms/iter + ... overlapped with compute: 58.415 ms/iter ( 53.4 %) + ... exposed on critical path: 50.914 ms/iter ( 46.6 %) + other GPU work (union) : 425.110 ms/iter + traced wall : 494.224 ms/iter + --> time hidden by overlap : 58.415 ms/iter diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..a84013223214e7e0a4ad900ebf74d3c450e202f5 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt @@ -0,0 +1 @@ +devices=8 reduce=109.246 ms/iter overlap=58.441 ms/iter (53.5%) exposed=50.805 ms/iter status=PASS diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log index 479f2c151b1c9fc81ea2522f01a5588be8d76e36..09d3362f1227a28afb6ae5b0dbc93713ae7dcbab 100644 --- a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log @@ -865,3 +865,68 @@ CUDA graph capture done!!! [2026-08-30 09:33:41] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 500.8 | throughput per GPU (TFLOP/s/GPU): 665.7 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.012681E+01 | loss scale: 1.0 | grad norm: 43.423 | number of skipped iterations: 0 | number of nan iterations: 0 | [2026-08-30 09:33:44] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3206.3 | throughput per GPU (TFLOP/s/GPU): 104.0 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.871505E+00 | loss scale: 1.0 | grad norm: 7.989 | number of skipped iterations: 0 | number of nan iterations: 0 | [2026-08-30 09:33:44] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 487.6 | throughput per GPU (TFLOP/s/GPU): 683.7 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.080805E+00 | loss scale: 1.0 | grad norm: 6.615 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture range started in the application. +Capture range ended in the application. +Generating '/tmp/nsys-report-c110.qdstrm' + [1/1] [0% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-repProcessing events... + [1/1] [0% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [7% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [10% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [13% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=16% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==20% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===23% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [====26% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=====29% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=====32% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [======35% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======39% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========40% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========41% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========43% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========44% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========46% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==========47% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==========49% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========51% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========53% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============54% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============56% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============57% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=============59% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==============61% ] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========================100%] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========================100%] G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep + [2026-08-30 09:34:27] iteration 9/ 10 | consumed samples: 144 | elapsed time per iteration (ms): 42485.7 | throughput per GPU (TFLOP/s/GPU): 7.8 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.706870E+00 | loss scale: 1.0 | grad norm: 6.800 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:34:27] iteration 10/ 10 | consumed samples: 160 | elapsed time per iteration (ms): 508.0 | throughput per GPU (TFLOP/s/GPU): 656.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 8.508333E+00 | loss scale: 1.0 | grad norm: 5.796 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-30 09:34:27 +[rank0]:[W830 09:34:29.253133092 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W830 09:34:30.434756072 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W830 09:34:30.439653428 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 09:34:30.501548768 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W830 09:34:30.710457488 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W830 09:34:30.894298920 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W830 09:34:30.062143723 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W830 09:34:31.620052340 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W830 09:34:33.744760822 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator()) +pure virtual method called +terminate called without an active exception +[rank0]:[W830 09:34:33.901720434 observer.cpp:116] Warning: Leaked callback handle: 1 (function operator()) +pure virtual method called +terminate called without an active exception +Collection stop is not allowed in this state. +W0830 09:34:34.074000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933075 closing signal SIGTERM +W0830 09:34:34.074000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933076 closing signal SIGTERM +W0830 09:34:34.075000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933077 closing signal SIGTERM +W0830 09:34:34.076000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933079 closing signal SIGTERM +W0830 09:34:34.078000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933080 closing signal SIGTERM +W0830 09:34:34.079000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933081 closing signal SIGTERM +W0830 09:34:34.080000 932866 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 933082 closing signal SIGTERM +E0830 09:34:35.739000 932866 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 3 (pid: 933078) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-30_09:34:34 + host : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 + rank : 3 (local_rank: 3) + exitcode : -6 (pid: 933078) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 933078 +============================================================ +Generated: + /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log new file mode 100644 index 0000000000000000000000000000000000000000..1fa13d7646a07381398ceb2147be6bb5ed3943f4 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log @@ -0,0 +1,6 @@ +report : /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +kernels : 127130 +devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7] +processes: 1 +markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7] +PASS: all ranks present and backend markers emitted diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep new file mode 100644 index 0000000000000000000000000000000000000000..94188be96d4e685f167223d36895aabed6754dea --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd9f456bc11bcad68e1f2d55a124e50fcad6be13301cff2443f566104fa392d +size 10443380 diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json new file mode 100644 index 0000000000000000000000000000000000000000..14df34ee550fbc33700881edf9df132e9bafe99d --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json @@ -0,0 +1,9 @@ +{ + "devices": 8, + "reduce_busy_ms_per_iteration": 28.751875, + "overlapped_ms_per_iteration": 27.93925, + "exposed_ms_per_iteration": 0.812625, + "overlap_pct": 97.17366258722258, + "minimum_overlap_pct": 0.1, + "status": "PASS" +} diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt new file mode 100644 index 0000000000000000000000000000000000000000..9b944d6dc952c415030ecdac1b1511acbb66c91b --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt @@ -0,0 +1,416 @@ + +=== kernels on device 0, by total time === + 222.064 ms device_kernel + 94.247 ms elementwise_kernel + 58.260 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.132 ms multi_tensor_apply_kernel + 48.102 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 46.940 ms cast_mxfp8_2D_kernel + 37.835 ms vectorized_elementwise_kernel + REDUCE 37.816 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 29.767 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.990 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 25.630 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 24.873 ms triton_poi_fused__to_copy_cat_0 + REDUCE 20.247 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 19.705 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 16.659 ms unrolled_elementwise_kernel + 12.134 ms ncclDevKernel_AllGather_RING_LL + 9.957 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 9.474 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.859 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.218 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.734 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.919 ms triton_poi_fused__to_copy_mul_silu_0 + 5.442 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.301 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.847 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 4.340 ms setGroupedGemmWgradArguments + 3.950 ms rmsnorm_bwd_tuned_kernel + 3.939 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.838 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.814 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.564 ms CatArrayBatchedCopy_alignedK_contig + 3.496 ms reduce_kernel + 3.397 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.390 ms _scatter_gather_elementwise_kernel + 3.212 ms multi_tensor_swizzle_row_scaling_kernel + 2.925 ms fused_rope_backward_kernel + 2.766 ms multi_tensor_swizzle_col_scaling_kernel + 2.458 ms fused_rope_forward_kernel + 2.296 ms swizzle_col_scaling_kernel + 1.944 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0 + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 29.032 ms/iter + reduce_grad busy (union) : 29.032 ms/iter + ... overlapped with compute: 28.236 ms/iter ( 97.3 %) + ... exposed on critical path: 0.795 ms/iter ( 2.7 %) + other GPU work (union) : 406.726 ms/iter + traced wall : 444.786 ms/iter + --> time hidden by overlap : 28.236 ms/iter + +=== kernels on device 1, by total time === + 218.011 ms device_kernel + 96.436 ms elementwise_kernel + 58.112 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.773 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 48.187 ms multi_tensor_apply_kernel + 46.749 ms cast_mxfp8_2D_kernel + 37.711 ms vectorized_elementwise_kernel + REDUCE 37.280 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 29.240 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.544 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 25.457 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 24.726 ms triton_poi_fused__to_copy_cat_0 + 21.229 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 19.806 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.659 ms unrolled_elementwise_kernel + 10.405 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 9.498 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 9.148 ms ncclDevKernel_AllGather_RING_LL + 8.830 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.119 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.641 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.980 ms triton_poi_fused__to_copy_mul_silu_0 + 5.968 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 5.418 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.227 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.460 ms setGroupedGemmWgradArguments + 3.970 ms rmsnorm_bwd_tuned_kernel + 3.921 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.905 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.800 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.557 ms CatArrayBatchedCopy_alignedK_contig + 3.482 ms reduce_kernel + 3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.319 ms _scatter_gather_elementwise_kernel + 3.284 ms multi_tensor_swizzle_row_scaling_kernel + 2.925 ms fused_rope_backward_kernel + 2.786 ms multi_tensor_swizzle_col_scaling_kernel + 2.451 ms fused_rope_forward_kernel + 2.253 ms swizzle_col_scaling_kernel + 1.988 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0 + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.543 ms/iter + reduce_grad busy (union) : 28.543 ms/iter + ... overlapped with compute: 27.703 ms/iter ( 97.1 %) + ... exposed on critical path: 0.840 ms/iter ( 2.9 %) + other GPU work (union) : 405.442 ms/iter + traced wall : 443.732 ms/iter + --> time hidden by overlap : 27.703 ms/iter + +=== kernels on device 2, by total time === + 216.307 ms device_kernel + 96.416 ms elementwise_kernel + 58.084 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.194 ms multi_tensor_apply_kernel + 48.124 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 46.869 ms cast_mxfp8_2D_kernel + 37.953 ms vectorized_elementwise_kernel + REDUCE 36.718 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 29.144 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.821 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 26.502 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 24.724 ms triton_poi_fused__to_copy_cat_0 + 21.168 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 20.391 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.773 ms unrolled_elementwise_kernel + 10.313 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 10.204 ms ncclDevKernel_AllGather_RING_LL + 9.486 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.838 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.212 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.758 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.947 ms triton_poi_fused__to_copy_mul_silu_0 + 6.377 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 5.442 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.263 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.315 ms setGroupedGemmWgradArguments + 3.960 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.945 ms rmsnorm_bwd_tuned_kernel + 3.848 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.837 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.552 ms CatArrayBatchedCopy_alignedK_contig + 3.519 ms reduce_kernel + 3.353 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.344 ms _scatter_gather_elementwise_kernel + 3.225 ms multi_tensor_swizzle_row_scaling_kernel + 2.835 ms fused_rope_backward_kernel + 2.743 ms multi_tensor_swizzle_col_scaling_kernel + 2.455 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0 + 2.452 ms fused_rope_forward_kernel + 2.279 ms swizzle_col_scaling_kernel + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.554 ms/iter + reduce_grad busy (union) : 28.554 ms/iter + ... overlapped with compute: 27.734 ms/iter ( 97.1 %) + ... exposed on critical path: 0.820 ms/iter ( 2.9 %) + other GPU work (union) : 406.207 ms/iter + traced wall : 443.971 ms/iter + --> time hidden by overlap : 27.734 ms/iter + +=== kernels on device 3, by total time === + 219.931 ms device_kernel + 96.310 ms elementwise_kernel + 58.117 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.923 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 48.072 ms multi_tensor_apply_kernel + 46.855 ms cast_mxfp8_2D_kernel + 37.473 ms vectorized_elementwise_kernel + REDUCE 36.402 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 29.275 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 28.747 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 27.786 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.749 ms triton_poi_fused__to_copy_cat_0 + REDUCE 20.723 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 17.471 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 16.621 ms unrolled_elementwise_kernel + 10.273 ms ncclDevKernel_AllGather_RING_LL + 9.475 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.824 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.582 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 8.193 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.821 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.990 ms triton_poi_fused__to_copy_mul_silu_0 + 6.932 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 5.464 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.214 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.492 ms setGroupedGemmWgradArguments + 3.980 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.963 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.930 ms rmsnorm_bwd_tuned_kernel + 3.840 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.535 ms CatArrayBatchedCopy_alignedK_contig + 3.486 ms reduce_kernel + 3.451 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.308 ms _scatter_gather_elementwise_kernel + 3.203 ms multi_tensor_swizzle_row_scaling_kernel + 2.867 ms fused_rope_backward_kernel + 2.791 ms multi_tensor_swizzle_col_scaling_kernel + 2.459 ms fused_rope_forward_kernel + 2.265 ms swizzle_col_scaling_kernel + 1.906 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0 + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.562 ms/iter + reduce_grad busy (union) : 28.562 ms/iter + ... overlapped with compute: 27.750 ms/iter ( 97.2 %) + ... exposed on critical path: 0.812 ms/iter ( 2.8 %) + other GPU work (union) : 406.509 ms/iter + traced wall : 443.946 ms/iter + --> time hidden by overlap : 27.750 ms/iter + +=== kernels on device 4, by total time === + 221.823 ms device_kernel + 96.605 ms elementwise_kernel + 58.051 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.697 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 48.199 ms multi_tensor_apply_kernel + 46.896 ms cast_mxfp8_2D_kernel + REDUCE 37.843 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 37.585 ms vectorized_elementwise_kernel + 29.295 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.894 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.762 ms triton_poi_fused__to_copy_cat_0 + 24.376 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 19.707 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.800 ms unrolled_elementwise_kernel + 16.589 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 11.394 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 10.496 ms ncclDevKernel_AllGather_RING_LL + 9.709 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.830 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.232 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.784 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.022 ms triton_poi_fused__to_copy_mul_silu_0 + 5.525 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.282 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.421 ms setGroupedGemmWgradArguments + 3.990 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.964 ms rmsnorm_bwd_tuned_kernel + 3.940 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.862 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.536 ms CatArrayBatchedCopy_alignedK_contig + 3.476 ms reduce_kernel + 3.443 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.309 ms _scatter_gather_elementwise_kernel + 3.294 ms multi_tensor_swizzle_row_scaling_kernel + 2.841 ms fused_rope_backward_kernel + 2.796 ms multi_tensor_swizzle_col_scaling_kernel + 2.462 ms fused_rope_forward_kernel + 2.273 ms swizzle_col_scaling_kernel + 1.876 ms rmsnorm_fwd_tuned_kernel + 1.777 ms gatherTopK + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.775 ms/iter + reduce_grad busy (union) : 28.775 ms/iter + ... overlapped with compute: 27.984 ms/iter ( 97.3 %) + ... exposed on critical path: 0.791 ms/iter ( 2.7 %) + other GPU work (union) : 403.102 ms/iter + traced wall : 444.881 ms/iter + --> time hidden by overlap : 27.984 ms/iter + +=== kernels on device 5, by total time === + 217.680 ms device_kernel + 96.334 ms elementwise_kernel + 58.085 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.507 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 48.151 ms multi_tensor_apply_kernel + 46.770 ms cast_mxfp8_2D_kernel + 37.545 ms vectorized_elementwise_kernel + REDUCE 36.084 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 35.700 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 29.428 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.711 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.683 ms triton_poi_fused__to_copy_cat_0 + REDUCE 21.207 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.757 ms unrolled_elementwise_kernel + 15.098 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 12.401 ms ncclDevKernel_AllGather_RING_LL + 9.470 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.827 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.198 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.712 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.661 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 6.966 ms triton_poi_fused__to_copy_mul_silu_0 + 5.443 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.231 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 5.172 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 4.336 ms setGroupedGemmWgradArguments + 3.938 ms rmsnorm_bwd_tuned_kernel + 3.906 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.874 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.840 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.523 ms CatArrayBatchedCopy_alignedK_contig + 3.470 ms reduce_kernel + 3.383 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.308 ms _scatter_gather_elementwise_kernel + 3.151 ms multi_tensor_swizzle_row_scaling_kernel + 2.842 ms fused_rope_backward_kernel + 2.754 ms multi_tensor_swizzle_col_scaling_kernel + 2.449 ms fused_rope_forward_kernel + 2.260 ms swizzle_col_scaling_kernel + 1.863 ms rmsnorm_fwd_tuned_kernel + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.645 ms/iter + reduce_grad busy (union) : 28.645 ms/iter + ... overlapped with compute: 27.822 ms/iter ( 97.1 %) + ... exposed on critical path: 0.824 ms/iter ( 2.9 %) + other GPU work (union) : 406.587 ms/iter + traced wall : 445.207 ms/iter + --> time hidden by overlap : 27.822 ms/iter + +=== kernels on device 6, by total time === + 216.806 ms device_kernel + 96.363 ms elementwise_kernel + 57.991 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.231 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 48.211 ms multi_tensor_apply_kernel + 46.817 ms cast_mxfp8_2D_kernel + 39.392 ms vectorized_elementwise_kernel + REDUCE 36.916 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 34.520 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 30.079 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.861 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.711 ms triton_poi_fused__to_copy_cat_0 + REDUCE 21.035 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.825 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 16.555 ms unrolled_elementwise_kernel + 11.149 ms ncclDevKernel_AllGather_RING_LL + 9.399 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.915 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 8.827 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.185 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.675 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.996 ms triton_poi_fused__to_copy_mul_silu_0 + 6.641 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 5.400 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.195 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.437 ms setGroupedGemmWgradArguments + 3.971 ms rmsnorm_bwd_tuned_kernel + 3.900 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.887 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.808 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.565 ms CatArrayBatchedCopy_alignedK_contig + 3.457 ms reduce_kernel + 3.392 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.307 ms _scatter_gather_elementwise_kernel + 3.132 ms multi_tensor_swizzle_row_scaling_kernel + 2.861 ms fused_rope_backward_kernel + 2.774 ms multi_tensor_swizzle_col_scaling_kernel + 2.460 ms fused_rope_forward_kernel + 2.241 ms swizzle_col_scaling_kernel + 1.855 ms rmsnorm_fwd_tuned_kernel + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.975 ms/iter + reduce_grad busy (union) : 28.975 ms/iter + ... overlapped with compute: 28.160 ms/iter ( 97.2 %) + ... exposed on critical path: 0.815 ms/iter ( 2.8 %) + other GPU work (union) : 407.996 ms/iter + traced wall : 484.104 ms/iter + --> time hidden by overlap : 28.160 ms/iter + +=== kernels on device 7, by total time === + 215.091 ms device_kernel + 96.345 ms elementwise_kernel + 58.110 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o2703364096i64i64_tensorptri32gmemalign16o327681_tensorptri32_0 + 48.206 ms multi_tensor_apply_kernel + 47.300 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o2703364096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o327681_tensorptri32gm_0 + 46.736 ms cast_mxfp8_2D_kernel + REDUCE 37.555 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 37.336 ms vectorized_elementwise_kernel + 35.418 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 30.342 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 27.735 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.703 ms triton_poi_fused__to_copy_cat_0 + REDUCE 20.303 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o1310723584i64i64_tensorptrf32gmemalign16o1310723584i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 17.240 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 16.659 ms unrolled_elementwise_kernel + 12.084 ms ncclDevKernel_AllGather_RING_LL + 9.517 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.825 ms triton_red_fused__to_copy_mul_silu_sum_1 + 8.545 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o131072358411i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 8.140 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.712 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.955 ms triton_poi_fused__to_copy_mul_silu_0 + 6.666 ms ncclDevKernel_AllReduce_Sum_f32_RING_LL + 5.466 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.190 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.314 ms setGroupedGemmWgradArguments + 3.959 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_321_tensorptri32_gmem_align16_o_327681_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.941 ms rmsnorm_bwd_tuned_kernel + 3.872 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.827 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.536 ms CatArrayBatchedCopy_alignedK_contig + 3.480 ms reduce_kernel + 3.376 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.309 ms _scatter_gather_elementwise_kernel + 3.233 ms multi_tensor_swizzle_row_scaling_kernel + 2.835 ms fused_rope_backward_kernel + 2.750 ms multi_tensor_swizzle_col_scaling_kernel + 2.475 ms fused_rope_forward_kernel + 2.304 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o337844096i64i64_tensorptri32gmemalign16o1013521_tensorptri32gmemalign16o337841_tensor_0 + 2.263 ms swizzle_col_scaling_kernel + +=== G2__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 28.929 ms/iter + reduce_grad busy (union) : 28.929 ms/iter + ... overlapped with compute: 28.125 ms/iter ( 97.2 %) + ... exposed on critical path: 0.804 ms/iter ( 2.8 %) + other GPU work (union) : 407.284 ms/iter + traced wall : 444.803 ms/iter + --> time hidden by overlap : 28.125 ms/iter diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..e7f0710d8ecad4ec903fec55d8b7d299d4b833ee --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt @@ -0,0 +1 @@ +devices=8 reduce=28.752 ms/iter overlap=27.939 ms/iter (97.2%) exposed=0.813 ms/iter status=PASS diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env new file mode 100644 index 0000000000000000000000000000000000000000..4a9980a39a5e9cbc05829680673710735490e657 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=G2 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce +commit=2e68c8ad +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=3584 +num_experts=32 +topk=8 +experts_per_rank=4 +expert_params=44040192 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=32768 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=4 +capacity_mode=none +capacity=none +cuda_graph=1 +probe_mode=0 +async_grad_reduce=1 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=10 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=1 +kernel_trace=1 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fbf372d01d9c081c5dcfd1393a313a18703a7429 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log @@ -0,0 +1,1450 @@ +============================================================== + G2 / moonep (slots/rank=4 policy=matched) + E=32 K=8 H'=3584 m=4 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=none cap=none cg=1 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad +============================================================== +W0830 09:35:17.183000 937001 torch/distributed/run.py:766] +W0830 09:35:17.183000 937001 torch/distributed/run.py:766] ***************************************** +W0830 09:35:17.183000 937001 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 09:35:17.183000 937001 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 3584 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 32 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... True + profile_ranks ................................... [0, 1, 2, 3, 4, 5, 6, 7] + profile_step_end ................................ 8 + profile_step_start .............................. 6 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 10 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 4 is connected to 7[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 75 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 76 is connected to 7 peer ranks. +Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 32 is connected to is connected to 7 peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 61 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W830 09:35:31.299314944 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 09:35:31.299314593 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 09:35:31.304520331 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 09:35:31.304611053 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 09:35:31.305521526 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 09:35:31.305658089 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 09:35:31.305986258 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.427 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 09:35:31.923129762 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.617 seconds +time to initialize megatron (seconds): 9.629 +[after megatron is initialized] datetime: 2026-08-30 09:35:37 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3903983616 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42086400 elements, 42086400 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42082304 elements, 42082304 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.weight +Params for bucket 4 (42082304 elements, 42082304 padded size): + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight +Params for bucket 5 (42082304 elements, 42082304 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight +Params for bucket 6 (42082304 elements, 42082304 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight +Params for bucket 7 (42082304 elements, 42082304 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight +Params for bucket 8 (42082304 elements, 42082304 padded size): + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 9 (42082304 elements, 42082304 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 10 (42082304 elements, 42082304 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 11 (42082304 elements, 42082304 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42082304 elements, 42082304 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight +Params for bucket 13 (42082304 elements, 42082304 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42082304 elements, 42082304 padded size): + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 15 (42082304 elements, 42082304 padded size): + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 16 (42082304 elements, 42082304 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42082304 elements, 42082304 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 56 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 +Params for bucket 3 (58720256 elements, 58720256 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 +Params for bucket 6 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (58720256 elements, 58720256 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (58720256 elements, 58720256 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 13 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 +Params for bucket 14 (58720256 elements, 58720256 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 17 (58720256 elements, 58720256 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 +Params for bucket 20 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 21 (58720256 elements, 58720256 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 24 (58720256 elements, 58720256 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 +Params for bucket 27 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 28 (58720256 elements, 58720256 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 31 (58720256 elements, 58720256 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 35 (58720256 elements, 58720256 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 38 (58720256 elements, 58720256 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 41 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 42 (58720256 elements, 58720256 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 45 (58720256 elements, 58720256 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 48 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 49 (58720256 elements, 58720256 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 52 (58720256 elements, 58720256 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 55 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 56 (58720256 elements, 58720256 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 09:35:40 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 160 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 09:35:40 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (2914.25, 2915.94) + train/valid/test-data-iterators-setup ..........: (88.46, 103.35) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 09:35:40 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=4096, E=32, K=8, NvS=33784, NvS_padded=33792 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 264.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 1.25 MiB + total_size : 267.25 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [1152, 21760, 6784, 3200, 128, 0, 0, 0] | duplicated experts [[4, -1, -1, -1], [-1, -1, -1, -1], [23, -1, -1, -1], [22, -1, -1, -1], [31, -1, -1, -1], [30, -1, -1, -1], [1, -1, -1, -1], [4, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [6528, 128, 11392, 896, 14208, 0, 0, 0] | duplicated experts [[4, -1, -1, -1], [28, -1, -1, -1], [19, -1, -1, -1], [28, -1, -1, -1], [13, -1, -1, -1], [27, -1, -1, -1], [7, -1, -1, -1], [-1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [9984, 11264, 768, 11136, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1], [23, -1, -1, -1], [3, -1, -1, -1], [3, -1, -1, -1], [0, -1, -1, -1], [13, -1, -1, -1], [17, -1, -1, -1], [6, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 32, 33, 34, 35] sizes [13184, 4480, 384, 14208, 768, 0, 0, 0] | duplicated experts [[19, -1, -1, -1], [12, -1, -1, -1], [17, -1, -1, -1], [0, -1, -1, -1], [-1, -1, -1, -1], [0, -1, -1, -1], [12, -1, -1, -1], [3, -1, -1, -1]] + [2026-08-30 09:36:00] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 20014.6 | throughput per GPU (TFLOP/s/GPU): 16.7 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102610E+01 | loss scale: 1.0 | grad norm: 44.466 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 53085.68505859375 | max allocated: 53085.685546875 | reserved: 56294.0 | max reserved: 56294.0 + [2026-08-30 09:36:01] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 507.9 | throughput per GPU (TFLOP/s/GPU): 656.4 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101605E+01 | loss scale: 1.0 | grad norm: 45.877 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:36:01] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 486.8 | throughput per GPU (TFLOP/s/GPU): 684.9 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.028761E+01 | loss scale: 1.0 | grad norm: 201.438 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-30 09:36:03] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 1335.3 | throughput per GPU (TFLOP/s/GPU): 249.7 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.322926E+01 | loss scale: 1.0 | grad norm: 31.649 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:36:03] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 434.2 | throughput per GPU (TFLOP/s/GPU): 767.9 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.039274E+01 | loss scale: 1.0 | grad norm: 16.281 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:36:04] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 429.7 | throughput per GPU (TFLOP/s/GPU): 775.9 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.047913E+01 | loss scale: 1.0 | grad norm: 38.985 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:36:07] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3163.9 | throughput per GPU (TFLOP/s/GPU): 105.4 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.013135E+01 | loss scale: 1.0 | grad norm: 7.751 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:36:07] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 440.8 | throughput per GPU (TFLOP/s/GPU): 756.4 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.560821E+00 | loss scale: 1.0 | grad norm: 5.845 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture range started in the application. +Capture range ended in the application. +Generating '/tmp/nsys-report-5c59.qdstrm' + [1/1] [0% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-repProcessing events... + [1/1] [0% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [6% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [9% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [12% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [14% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=17% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==20% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===23% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [====26% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=====29% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=====31% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [======34% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======36% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======37% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======39% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========40% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========41% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========42% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========43% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========45% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========46% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==========48% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==========49% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========51% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========52% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============54% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============55% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============57% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=============60% ] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========================100%] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========================100%] G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +MoonEP cross_rank_barrier timeout (100s), trapping: rank=3 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=5 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=1 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=2 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank5]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank5]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: RuntimeError: CUDA error: unspecified launch failure +[rank5]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank5]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank5]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank1]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank1]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: RuntimeError: CUDA error: unspecified launch failure +[rank1]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank1]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank1]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=7 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank3]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank3]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: RuntimeError: CUDA error: unspecified launch failure +[rank3]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank3]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank3]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=6 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +[rank7]: Traceback (most recent call last): +[rank7]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank7]: pretrain( +[rank7]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank7]: iteration, num_floating_point_operations_so_far = train( +[rank7]: ^^^^^^ +[rank7]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank7]: ) = train_step( +[rank7]: ^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank7]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank7]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank7]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank7]: ^^^^^^^^^^^^^ +[rank7]: RuntimeError: CUDA error: unspecified launch failure +[rank7]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank7]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank7]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank2]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank2]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: RuntimeError: CUDA error: unspecified launch failure +[rank2]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank2]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank2]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank6]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank6]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: RuntimeError: CUDA error: unspecified launch failure +[rank6]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank6]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank6]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=0 barrier_off=134024 phase=0 sign=0 signal=7 target=8 + +[rank0]: Traceback (most recent call last): +[rank0]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank0]: pretrain( +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank0]: ) = train_step( +[rank0]: ^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank0]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank0]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: RuntimeError: CUDA error: unspecified launch failure +[rank0]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank0]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank0]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x72b100bd9489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x72b100bdb568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x72b100b77221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x72b100c98c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x72b08667c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x72b0866778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x72b08667f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x72b0e943ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x72b100bb8409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x72b0e96f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x72b0e96f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x72b1052991ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x72b10529928b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x705ed4392489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x705ed4394568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x705ed4330221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x705ed73d3c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x705e5d6958fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x705e5d6908cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x705e5d698084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x705ecf23ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x705ed4371409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x705ecf4f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x705ecf4f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x705ede7f61ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x705ede7f628b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x7b7a36c79489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x7b7a36c7b568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x7b7a36c17221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x7b7ab007fc82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x7b7a378958fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x7b7a378908cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x7b7a37898084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x7b7ab051fc52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x7b7a36c58409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x7b7ab07d9688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x7b7ab07d9ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x7b7ab57e11ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x7b7ab57e128b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x76b1f45d9489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x76b1f45db568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x76b1f4577221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x76b1f4698c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x76b17a67c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x76b17a6778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x76b17a67f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x76b1dd43ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x76b1f45b8409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x76b1dd6f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x76b1dd6f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x76b1f8c1a1ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x76b1f8c1a28b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x72445cfd9489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x72445cfdb568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x72445cf77221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x72445d098c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x7243e267c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x7243e26778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x7243e267f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x72444543ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x72445cfb8409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x7244456f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x7244456f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x7244615e61ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x7244615e628b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x70fe6e192489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x70fe6e194568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x70fe6e130221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x70fe84098c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x70fe0a27c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x70fe0a2778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x70fe0a27f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x70fe6d03ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x70fe6e171409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x70fe6d2f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x70fe6d2f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x70fe883521ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x70fe8835228b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x73c6242db489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x73c6242dd568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x73c624279221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x73c62439ac82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x73c5ab8958fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x73c5ab8908cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x73c5ab898084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x73c62483ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x73c6242ba409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x73c624af4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x73c624af4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x73c629b791ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x73c629b7928b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +W0830 09:38:44.449000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937215 closing signal SIGTERM +W0830 09:38:44.450000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937216 closing signal SIGTERM +W0830 09:38:44.450000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937217 closing signal SIGTERM +W0830 09:38:44.451000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937218 closing signal SIGTERM +W0830 09:38:44.451000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937219 closing signal SIGTERM +W0830 09:38:44.455000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937221 closing signal SIGTERM +W0830 09:38:44.456000 937001 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 937222 closing signal SIGTERM +E0830 09:38:46.524000 937001 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 5 (pid: 937220) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-30_09:38:44 + host : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 + rank : 5 (local_rank: 5) + exitcode : -6 (pid: 937220) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 937220 +============================================================ +The target application terminated. One or more process it created re-parented. +Waiting for termination of re-parented processes. +Use the `--wait` option to modify this behavior. +Generated: + /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log new file mode 100644 index 0000000000000000000000000000000000000000..1f1c1cbb5497606db8ae2b322c19b21f5b5f71d3 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log @@ -0,0 +1,6 @@ +report : /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +kernels : 133157 +devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7] +processes: 1 +markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7] +PASS: all ranks present and backend markers emitted diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep new file mode 100644 index 0000000000000000000000000000000000000000..90ca120acb034baab1b321721b6759ff4b266be9 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df838cb4bcff056f30b96b8d1814b40ba9ec10475e43966ba8ef786ac5ecbae2 +size 11588634 diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json new file mode 100644 index 0000000000000000000000000000000000000000..b33c69384aff3be6114fd0d38bc83eba184c7f87 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json @@ -0,0 +1,9 @@ +{ + "devices": 8, + "reduce_busy_ms_per_iteration": 17.206125, + "overlapped_ms_per_iteration": 16.46425, + "exposed_ms_per_iteration": 0.742, + "overlap_pct": 95.68830866915124, + "minimum_overlap_pct": 0.1, + "status": "PASS" +} diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt new file mode 100644 index 0000000000000000000000000000000000000000..56cdc12c121a1305eb50efb08239784cabd71787 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.txt @@ -0,0 +1,416 @@ + +=== kernels on device 0, by total time === + 225.290 ms device_kernel + 175.772 ms elementwise_kernel + 60.753 ms cast_mxfp8_2D_kernel + 59.799 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 50.565 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 47.968 ms multi_tensor_apply_kernel + 47.645 ms vectorized_elementwise_kernel + 28.042 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 26.365 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 24.993 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.753 ms triton_poi_fused__to_copy_cat_0 + 24.705 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + REDUCE 23.330 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.295 ms unrolled_elementwise_kernel + 13.756 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + REDUCE 11.254 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 10.528 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 8.926 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.655 ms ncclDevKernel_AllGather_RING_LL + 8.277 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.837 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.778 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.925 ms triton_poi_fused__to_copy_mul_silu_0 + 6.235 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.168 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 4.980 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.198 ms multi_tensor_swizzle_row_scaling_kernel + 4.187 ms setGroupedGemmWgradArguments + 4.143 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 3.938 ms _scatter_gather_elementwise_kernel + 3.926 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.922 ms rmsnorm_bwd_tuned_kernel + 3.805 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.629 ms reduce_kernel + 3.471 ms CatArrayBatchedCopy_alignedK_contig + 3.388 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.004 ms gatherTopK + 2.833 ms multi_tensor_swizzle_col_scaling_kernel + 2.621 ms fused_rope_backward_kernel + 2.455 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 0, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.292 ms/iter + reduce_grad busy (union) : 17.292 ms/iter + ... overlapped with compute: 16.580 ms/iter ( 95.9 %) + ... exposed on critical path: 0.712 ms/iter ( 4.1 %) + other GPU work (union) : 469.865 ms/iter + traced wall : 660.128 ms/iter + --> time hidden by overlap : 16.580 ms/iter + +=== kernels on device 1, by total time === + 224.560 ms device_kernel + 180.765 ms elementwise_kernel + 60.594 ms cast_mxfp8_2D_kernel + 59.763 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 48.869 ms vectorized_elementwise_kernel + 48.758 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.204 ms multi_tensor_apply_kernel + 25.708 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.182 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 24.859 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.786 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.963 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 17.737 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 16.598 ms unrolled_elementwise_kernel + 13.843 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + 11.722 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 11.356 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 9.512 ms ncclDevKernel_AllGather_RING_LL + 9.031 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.225 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.820 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.719 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.014 ms triton_poi_fused__to_copy_mul_silu_0 + 6.119 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.191 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 4.987 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.302 ms multi_tensor_swizzle_row_scaling_kernel + 4.283 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.261 ms setGroupedGemmWgradArguments + 3.956 ms _scatter_gather_elementwise_kernel + 3.931 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.931 ms rmsnorm_bwd_tuned_kernel + 3.800 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.594 ms reduce_kernel + 3.456 ms CatArrayBatchedCopy_alignedK_contig + 3.370 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 2.998 ms gatherTopK + 2.871 ms multi_tensor_swizzle_col_scaling_kernel + 2.626 ms fused_rope_backward_kernel + 2.447 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 1, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.159 ms/iter + reduce_grad busy (union) : 17.159 ms/iter + ... overlapped with compute: 16.403 ms/iter ( 95.6 %) + ... exposed on critical path: 0.756 ms/iter ( 4.4 %) + other GPU work (union) : 468.435 ms/iter + traced wall : 630.040 ms/iter + --> time hidden by overlap : 16.403 ms/iter + +=== kernels on device 2, by total time === + 222.602 ms device_kernel + 180.628 ms elementwise_kernel + 60.950 ms cast_mxfp8_2D_kernel + 59.707 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 49.101 ms vectorized_elementwise_kernel + 48.859 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.325 ms multi_tensor_apply_kernel + 25.365 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.357 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 24.947 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.729 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.867 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 17.862 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 16.723 ms unrolled_elementwise_kernel + 13.896 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + 12.801 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 11.352 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 10.792 ms ncclDevKernel_AllGather_RING_LL + 9.070 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.304 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.849 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.822 ms triton_red_fused__to_copy_mul_silu_sum_1 + 6.960 ms triton_poi_fused__to_copy_mul_silu_0 + 6.097 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.278 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.067 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.291 ms multi_tensor_swizzle_row_scaling_kernel + 4.283 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.184 ms setGroupedGemmWgradArguments + 3.992 ms rmsnorm_bwd_tuned_kernel + 3.987 ms _scatter_gather_elementwise_kernel + 3.855 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.843 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.624 ms reduce_kernel + 3.466 ms CatArrayBatchedCopy_alignedK_contig + 3.373 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.008 ms gatherTopK + 2.815 ms multi_tensor_swizzle_col_scaling_kernel + 2.626 ms fused_rope_backward_kernel + 2.457 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 2, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.110 ms/iter + reduce_grad busy (union) : 17.110 ms/iter + ... overlapped with compute: 16.374 ms/iter ( 95.7 %) + ... exposed on critical path: 0.736 ms/iter ( 4.3 %) + other GPU work (union) : 469.923 ms/iter + traced wall : 660.796 ms/iter + --> time hidden by overlap : 16.374 ms/iter + +=== kernels on device 3, by total time === + 228.029 ms device_kernel + 180.601 ms elementwise_kernel + 60.977 ms cast_mxfp8_2D_kernel + 59.761 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 49.678 ms vectorized_elementwise_kernel + 49.237 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.234 ms multi_tensor_apply_kernel + 25.169 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 25.044 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.896 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 24.787 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.655 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.728 ms unrolled_elementwise_kernel + 13.867 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + 12.902 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 11.896 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 11.669 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 8.988 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.917 ms ncclDevKernel_AllGather_RING_LL + 8.285 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.892 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.820 ms triton_red_fused__to_copy_mul_silu_sum_1 + 6.999 ms triton_poi_fused__to_copy_mul_silu_0 + 5.768 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.216 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.026 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.285 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.278 ms multi_tensor_swizzle_row_scaling_kernel + 4.145 ms setGroupedGemmWgradArguments + 3.997 ms _scatter_gather_elementwise_kernel + 3.968 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.948 ms rmsnorm_bwd_tuned_kernel + 3.840 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.675 ms reduce_kernel + 3.465 ms CatArrayBatchedCopy_alignedK_contig + 3.439 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.011 ms gatherTopK + 2.870 ms multi_tensor_swizzle_col_scaling_kernel + 2.625 ms fused_rope_backward_kernel + 2.449 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 3, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.162 ms/iter + reduce_grad busy (union) : 17.162 ms/iter + ... overlapped with compute: 16.412 ms/iter ( 95.6 %) + ... exposed on critical path: 0.750 ms/iter ( 4.4 %) + other GPU work (union) : 468.637 ms/iter + traced wall : 634.045 ms/iter + --> time hidden by overlap : 16.412 ms/iter + +=== kernels on device 4, by total time === + 228.089 ms device_kernel + 180.813 ms elementwise_kernel + 60.956 ms cast_mxfp8_2D_kernel + 59.664 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 48.814 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.233 ms multi_tensor_apply_kernel + 47.464 ms vectorized_elementwise_kernel + 25.377 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 25.019 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.960 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 24.804 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.213 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 16.756 ms unrolled_elementwise_kernel + 15.607 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 13.913 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + REDUCE 12.357 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 11.126 ms ncclDevKernel_AllGather_RING_LL + 9.766 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 9.194 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.316 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.846 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 7.817 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.017 ms triton_poi_fused__to_copy_mul_silu_0 + 5.302 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.110 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 5.063 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 4.289 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.220 ms setGroupedGemmWgradArguments + 4.210 ms multi_tensor_swizzle_row_scaling_kernel + 4.011 ms _scatter_gather_elementwise_kernel + 3.989 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.984 ms rmsnorm_bwd_tuned_kernel + 3.853 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.658 ms reduce_kernel + 3.472 ms CatArrayBatchedCopy_alignedK_contig + 3.423 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.014 ms gatherTopK + 2.881 ms multi_tensor_swizzle_col_scaling_kernel + 2.629 ms fused_rope_backward_kernel + 2.463 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 4, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.285 ms/iter + reduce_grad busy (union) : 17.285 ms/iter + ... overlapped with compute: 16.549 ms/iter ( 95.7 %) + ... exposed on critical path: 0.736 ms/iter ( 4.3 %) + other GPU work (union) : 468.284 ms/iter + traced wall : 584.115 ms/iter + --> time hidden by overlap : 16.549 ms/iter + +=== kernels on device 5, by total time === + 223.409 ms device_kernel + 180.662 ms elementwise_kernel + 60.780 ms cast_mxfp8_2D_kernel + 59.720 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 49.220 ms vectorized_elementwise_kernel + 49.112 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.218 ms multi_tensor_apply_kernel + 25.591 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.158 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 24.895 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.739 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.718 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 18.214 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 16.703 ms unrolled_elementwise_kernel + 13.817 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + 11.849 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + REDUCE 11.639 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 10.266 ms ncclDevKernel_AllGather_RING_LL + 9.080 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.270 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.821 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.752 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.980 ms triton_poi_fused__to_copy_mul_silu_0 + 5.730 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.234 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.087 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.290 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.239 ms multi_tensor_swizzle_row_scaling_kernel + 4.198 ms setGroupedGemmWgradArguments + 3.993 ms _scatter_gather_elementwise_kernel + 3.956 ms rmsnorm_bwd_tuned_kernel + 3.875 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.832 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.620 ms reduce_kernel + 3.457 ms CatArrayBatchedCopy_alignedK_contig + 3.374 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.020 ms gatherTopK + 2.833 ms multi_tensor_swizzle_col_scaling_kernel + 2.630 ms fused_rope_backward_kernel + 2.447 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 5, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.178 ms/iter + reduce_grad busy (union) : 17.178 ms/iter + ... overlapped with compute: 16.423 ms/iter ( 95.6 %) + ... exposed on critical path: 0.755 ms/iter ( 4.4 %) + other GPU work (union) : 469.125 ms/iter + traced wall : 659.490 ms/iter + --> time hidden by overlap : 16.423 ms/iter + +=== kernels on device 6, by total time === + 225.278 ms device_kernel + 180.770 ms elementwise_kernel + 60.786 ms cast_mxfp8_2D_kernel + 59.662 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 48.635 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.231 ms multi_tensor_apply_kernel + 46.978 ms vectorized_elementwise_kernel + 25.916 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.417 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 24.999 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.748 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.619 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 18.939 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 16.603 ms unrolled_elementwise_kernel + 13.962 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + REDUCE 11.866 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 10.605 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 10.150 ms ncclDevKernel_AllGather_RING_LL + 8.950 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.258 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.821 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.742 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.993 ms triton_poi_fused__to_copy_mul_silu_0 + 5.393 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.175 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 4.976 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.286 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.151 ms multi_tensor_swizzle_row_scaling_kernel + 4.109 ms setGroupedGemmWgradArguments + 3.962 ms _scatter_gather_elementwise_kernel + 3.950 ms rmsnorm_bwd_tuned_kernel + 3.888 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.808 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.621 ms reduce_kernel + 3.454 ms CatArrayBatchedCopy_alignedK_contig + 3.386 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.005 ms gatherTopK + 2.846 ms multi_tensor_swizzle_col_scaling_kernel + 2.631 ms fused_rope_backward_kernel + 2.452 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 6, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.242 ms/iter + reduce_grad busy (union) : 17.242 ms/iter + ... overlapped with compute: 16.477 ms/iter ( 95.6 %) + ... exposed on critical path: 0.765 ms/iter ( 4.4 %) + other GPU work (union) : 468.131 ms/iter + traced wall : 583.652 ms/iter + --> time hidden by overlap : 16.477 ms/iter + +=== kernels on device 7, by total time === + 223.053 ms device_kernel + 180.620 ms elementwise_kernel + 60.854 ms cast_mxfp8_2D_kernel + 59.672 ms kernel_cutlass_kernel_moonepdispatchDispatchKernel_object_at__tensorptrbf16gmemalign16o40964096i64i64_tensorptrbf16gmemalign16o5406724096i64i64_tensorptri32gmemalign16o655361_tensorptri32_0 + 48.698 ms kernel_cutlass_kernel_moonepcombineCombineKernel_object_at__tensorptrbf16gmemalign16o5406724096i64i64_tensorptrbf16gmemalign16o40964096i64i64_tensorptri32gmemalign16o655361_tensorptri32gm_0 + 48.292 ms multi_tensor_apply_kernel + 47.209 ms vectorized_elementwise_kernel + 25.902 ms ncclDevKernel_ReduceScatter_Sum_f32_RING_LL + 25.368 ms kernel_cutlass_kernel_moonepcombine_prologueCombinePrologueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tensor_0 + 24.804 ms cudnn_generated_fort_native_sdpa_sm100_flash_bprop_f16_knob_31_128x128x128_1x4x1_kernel0_0 + 24.734 ms triton_poi_fused__to_copy_cat_0 + REDUCE 22.758 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2293764096i64i64_tensorptrf32gmemalign16o2293764096i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 20.086 ms kernel_cutlass_kernel_moonepinter_rank_syncInterRankSyncKernel_object_at__tensorptri32gmemalign16o41943041_tensorptri32gmemalign16o10___0 + 16.769 ms unrolled_elementwise_kernel + 13.922 ms kernel_cutlass_kernel_moonepdispatch_epilogueDispatchEpilogueKernel_object_at__tensorptrbf16gmemalign16o675684096i64i64_tensorptri32gmemalign16o2027041_tensorptri32gmemalign16o675681_tens_0 + REDUCE 11.685 ms kernel_cutlass_kernel_moonepgrad_reduceGradReduceKernel_object_at__tensorptrf32gmemalign16o2621441792i64i64_tensorptrf32gmemalign16o2621441792i64i64_CopyAtom_ThrID10_TVLayoutSrc11638401_T_0 + 10.928 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o229376409611i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 8.968 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NNT + 8.420 ms ncclDevKernel_AllGather_RING_LL + 8.248 ms nvjet_qqtst_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_TNT + 7.820 ms triton_red_fused__to_copy_mul_silu_sum_1 + 7.793 ms cudnn_generated_fort_native_sdpa_sm100_flash_fprop_f16_knob_7_128x128x128_4x1x1_kernel0_0 + 6.966 ms triton_poi_fused__to_copy_mul_silu_0 + 6.195 ms kernel_cutlass_kernel_moonepprefetchPrefetchKernel_object_at__CopyAtom_ThrID10_TVLayoutSrc11638401_TVLayoutDst11638401_Valuetypebf16_tensor00o262144179211i640_CopyAtom_ThrID10_TVLayoutSrc_0 + 5.234 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_badd_Avec32UE8M0_Bvec32UE8M0_NTT + 5.026 ms nvjet_qqsss_128x256_128x6_2x2_2cta_h_bz_Avec32UE8M0_Bvec32UE8M0_NTT + 4.274 ms kernel_cutlass_kernel_moonepplanningPlanningKernel_object_at__tensorptri32_gmem_align16_o_641_tensorptri32_gmem_align16_o_655361_tensorptri32_gmem_align16_o_41943041_tensorptri32_gmem_ali_0 + 4.267 ms multi_tensor_swizzle_row_scaling_kernel + 4.140 ms setGroupedGemmWgradArguments + 4.015 ms _scatter_gather_elementwise_kernel + 3.906 ms rmsnorm_bwd_tuned_kernel + 3.865 ms nvjet_tss_128x256_64x6_2x1_2cta_v_badd_NTT + 3.837 ms nvjet_tst_256x256_64x4_2x2_2cta_h_bz_NNT + 3.630 ms reduce_kernel + 3.469 ms CatArrayBatchedCopy_alignedK_contig + 3.366 ms nvjet_tst_128x256_64x6_2x1_2cta_v_bz_TNT + 3.016 ms gatherTopK + 2.825 ms multi_tensor_swizzle_col_scaling_kernel + 2.627 ms fused_rope_backward_kernel + 2.452 ms fused_rope_forward_kernel + +=== G3__moonep__matched__seq4096__gbs16__capnone__async.sqlite (device 7, 2 iteration(s)) === + reduce_grad kernels : 128 (64/iter) + reduce_grad sum : 17.221 ms/iter + reduce_grad busy (union) : 17.221 ms/iter + ... overlapped with compute: 16.496 ms/iter ( 95.8 %) + ... exposed on critical path: 0.726 ms/iter ( 4.2 %) + other GPU work (union) : 467.542 ms/iter + traced wall : 582.845 ms/iter + --> time hidden by overlap : 16.496 ms/iter diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt new file mode 100644 index 0000000000000000000000000000000000000000..7ec0c0b348f46e1d294f290898c793a6a6f76400 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap_summary.txt @@ -0,0 +1 @@ +devices=8 reduce=17.206 ms/iter overlap=16.464 ms/iter (95.7%) exposed=0.742 ms/iter status=PASS diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env new file mode 100644 index 0000000000000000000000000000000000000000..c29b4bf12f2f6c4d71007b1e9e02c3a99017bb1e --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=G3 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce +commit=2e68c8ad +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=1792 +num_experts=64 +topk=16 +experts_per_rank=8 +expert_params=22020096 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=65536 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=8 +capacity_mode=none +capacity=none +cuda_graph=1 +probe_mode=0 +async_grad_reduce=1 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=10 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=1 +kernel_trace=1 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log new file mode 100644 index 0000000000000000000000000000000000000000..96ee6973a224179020c8cb9d4721e8cd4e511e45 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log @@ -0,0 +1,1583 @@ +============================================================== + G3 / moonep (slots/rank=8 policy=matched) + E=64 K=16 H'=1792 m=8 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=none cap=none cg=1 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad +============================================================== +W0830 09:39:09.920000 940602 torch/distributed/run.py:766] +W0830 09:39:09.920000 940602 torch/distributed/run.py:766] ***************************************** +W0830 09:39:09.920000 940602 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 09:39:09.920000 940602 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1792 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 16 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... True + profile_ranks ................................... [0, 1, 2, 3, 4, 5, 6, 7] + profile_step_end ................................ 8 + profile_step_start .............................. 6 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 10 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 45 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank7]:[W830 09:39:23.831593725 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 09:39:23.831601397 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 09:39:23.832627003 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 09:39:23.832723318 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 09:39:23.832723058 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W830 09:39:23.832795591 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 09:39:23.832898932 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.423 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 09:39:24.442984191 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.932 seconds +time to initialize megatron (seconds): 9.960 +[after megatron is initialized] datetime: 2026-08-30 09:39:30 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3906080768 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42217472 elements, 42217472 padded size): + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.pre_mlp_layernorm.weight +Params for bucket 3 (42213376 elements, 42213376 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.pre_mlp_layernorm.weight +Params for bucket 4 (42213376 elements, 42213376 padded size): + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 5 (42213376 elements, 42213376 padded size): + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 6 (42213376 elements, 42213376 padded size): + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.pre_mlp_layernorm.weight +Params for bucket 7 (42213376 elements, 42213376 padded size): + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42213376 elements, 42213376 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 9 (42213376 elements, 42213376 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 10 (42213376 elements, 42213376 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight +Params for bucket 11 (42213376 elements, 42213376 padded size): + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 12 (42213376 elements, 42213376 padded size): + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (42213376 elements, 42213376 padded size): + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 14 (42213376 elements, 42213376 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42213376 elements, 42213376 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight +Params for bucket 16 (42213376 elements, 42213376 padded size): + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight +Params for bucket 17 (42213376 elements, 42213376 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 64 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 5 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 +Params for bucket 6 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 +Params for bucket 9 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 +Params for bucket 10 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 +Params for bucket 17 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 +Params for bucket 18 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 30 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 31 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 35 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 +Params for bucket 42 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 +Params for bucket 43 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 47 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 55 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 +Params for bucket 56 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +Params for bucket 63 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 09:39:33 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 160 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 09:39:33 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3172.63, 3179.88) + train/valid/test-data-iterators-setup ..........: (83.50, 107.39) +[before the start of training step] datetime: 2026-08-30 09:39:33 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=4096, E=64, K=16, NvS=67568, NvS_padded=67584 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 528.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.26 MiB + total_size : 532.26 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [1024, 23296, 6528, 3072, 11648, 4480, 7936, 7936, 384, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[22, -1, -1, -1, -1, -1, -1, -1], [32, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [31, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1], [40, 45, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [21888, 640, 1280, 24448, 3840, 3072, 0, 128, 10752, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[36, -1, -1, -1, -1, -1, -1, -1], [6, 0, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1], [17, -1, -1, -1, -1, -1, -1, -1], [35, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [23168, 4480, 18048, 768, 4480, 128, 14592, 0, 256, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[57, -1, -1, -1, -1, -1, -1, -1], [43, 40, -1, -1, -1, -1, -1, -1], [46, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1], [7, 2, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 64, 65, 66, 67, 68, 69, 70, 71] sizes [3200, 128, 29312, 8832, 1536, 512, 896, 17024, 4736, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[62, -1, -1, -1, -1, -1, -1, -1], [56, -1, -1, -1, -1, -1, -1, -1], [40, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1], [54, -1, -1, -1, -1, -1, -1, -1]] + [2026-08-30 09:39:54] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 20861.1 | throughput per GPU (TFLOP/s/GPU): 16.0 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102347E+01 | loss scale: 1.0 | grad norm: 44.191 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 53065.97900390625 | max allocated: 53065.9794921875 | reserved: 55708.0 | max reserved: 55708.0 + [2026-08-30 09:39:55] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 621.0 | throughput per GPU (TFLOP/s/GPU): 536.8 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101568E+01 | loss scale: 1.0 | grad norm: 45.943 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:39:56] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 547.9 | throughput per GPU (TFLOP/s/GPU): 608.5 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.051990E+01 | loss scale: 1.0 | grad norm: 215.098 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-30 09:39:57] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 1470.5 | throughput per GPU (TFLOP/s/GPU): 226.7 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.315973E+01 | loss scale: 1.0 | grad norm: 38.085 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:39:58] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 560.7 | throughput per GPU (TFLOP/s/GPU): 594.5 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.038138E+01 | loss scale: 1.0 | grad norm: 16.246 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:39:58] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 563.2 | throughput per GPU (TFLOP/s/GPU): 592.0 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.038052E+01 | loss scale: 1.0 | grad norm: 32.671 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:40:01] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3305.3 | throughput per GPU (TFLOP/s/GPU): 100.9 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 1.004488E+01 | loss scale: 1.0 | grad norm: 7.310 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:40:02] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 582.6 | throughput per GPU (TFLOP/s/GPU): 572.3 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.519695E+00 | loss scale: 1.0 | grad norm: 5.763 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture range started in the application. +Capture range ended in the application. +Generating '/tmp/nsys-report-41da.qdstrm' + [1/1] [0% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-repProcessing events... + [1/1] [0% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [5% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [8% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [10% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [13% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=16% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==18% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==21% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===23% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [====26% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [====28% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=====31% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [======33% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [======35% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======36% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======37% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======38% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=======39% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========40% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========41% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========42% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========43% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========44% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [=========46% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==========47% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==========48% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========50% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========51% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [===========53% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============54% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============55% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [============57% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [==============61% ] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========================100%] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep [1/1] [========================100%] G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +MoonEP cross_rank_barrier timeout (100s), trapping: rank=4 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=5 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=3 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=1 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=2 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank5]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank5]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: RuntimeError: CUDA error: unspecified launch failure +[rank5]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank5]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank5]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank1]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank1]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: RuntimeError: CUDA error: unspecified launch failure +[rank1]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank1]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank1]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank3]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank3]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: RuntimeError: CUDA error: unspecified launch failure +[rank3]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank3]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank3]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank4]: Traceback (most recent call last): +[rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank4]: pretrain( +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank4]: iteration, num_floating_point_operations_so_far = train( +[rank4]: ^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank4]: ) = train_step( +[rank4]: ^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank4]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank4]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank4]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank4]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank4]: ^^^^^^^^^^^^^ +[rank4]: RuntimeError: CUDA error: unspecified launch failure +[rank4]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank4]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank4]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +[rank2]: Traceback (most recent call last): +[rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank2]: pretrain( +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank2]: iteration, num_floating_point_operations_so_far = train( +[rank2]: ^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank2]: ) = train_step( +[rank2]: ^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank2]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank2]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank2]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank2]: ^^^^^^^^^^^^^ +[rank2]: RuntimeError: CUDA error: unspecified launch failure +[rank2]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank2]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank2]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=6 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank6]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank6]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: RuntimeError: CUDA error: unspecified launch failure +[rank6]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank6]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank6]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +MoonEP cross_rank_barrier timeout (100s), trapping: rank=0 barrier_off=268032 phase=1 sign=0 signal=7 target=8 + +[rank0]: Traceback (most recent call last): +[rank0]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank0]: pretrain( +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: ^^^^^^ +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2337, in train +[rank0]: ) = train_step( +[rank0]: ^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1429, in train_step +[rank0]: while rerun_state_machine.should_run_forward_backward(data_iterator): +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/workspace/Megatron-MoE-Echo/megatron/core/rerun_state_machine.py", line 309, in should_run_forward_backward +[rank0]: will_rerun_tensor: torch.Tensor = torch.tensor( +[rank0]: ^^^^^^^^^^^^^ +[rank0]: RuntimeError: CUDA error: unspecified launch failure +[rank0]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +[rank0]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +[rank0]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x7adf005d9489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x7adf005db568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x7adf00577221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x7adf00698c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x7ade8667c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x7ade866778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x7ade8667f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x7adee943ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x7adf005b8409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x7adee96f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x7adee96f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x7adf04bf01ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x7adf04bf028b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x7c87e0392489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x7c87e0394568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x7c87e0330221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x7c87e33d3c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x7c87696958fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x7c87696908cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x7c8769698084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x7c87db23ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x7c87e0371409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x7c87db4f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x7c87db4f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x7c87ea7f61ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x7c87ea7f628b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x79f9149d9489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x79f9149db568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x79f914977221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x79f914a98c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x79f89a67c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x79f89a6778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x79f89a67f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x79f8fd43ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x79f9149b8409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x79f8fd6f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x79f8fd6f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x79f9190331ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x79f91903328b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x78ed00592489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x78ed00594568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x78ed00530221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x78ed027a9c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x78ec8c47c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x78ec8c4778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x78ec8c47f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x78ecef23ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x78ed00571409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x78ecef4f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x78ecef4f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x78ed0a8de1ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x78ed0a8de28b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x766ba01aa489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x766ba01ac568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x766ba0148221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x766ba0269c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x766b2587c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x766b258778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x766b2587f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x766b8863ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x766ba0189409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x766b888f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x766b888f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x766ba3a351ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x766ba3a3528b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x75cee3e60489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x75cee3e62568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x75cee3dfe221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x75cf53590c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x75cee4a7c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x75cee4a778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x75cee4a7f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x75cf4783ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x75cee3e3f409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x75cf47af4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x75cf47af4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x75cf629841ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x75cf6298428b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +terminate called after throwing an instance of 'c10::Error' + what(): CUDA error: unspecified launch failure +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1 +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. + +Exception raised from c10_cuda_check_implementation at /opt/pytorch/pytorch/c10/cuda/CUDAException.cpp:43 (most recent call first): +frame #0: + 0x90489 (0x73a8f4460489 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #1: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string, std::allocator >) + 0x88 (0x73a8f4462568 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #2: c10::detail::torchCheckFail(char const*, char const*, unsigned int, std::__cxx11::basic_string, std::allocator > const&) + 0x55 (0x73a8f43fe221 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #3: c10::cuda::c10_cuda_check_implementation(int, char const*, char const*, int, bool) + 0x3e2 (0x73a962db3c82 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10_cuda.so) +frame #4: + 0xbae8fb (0x73a8f507c8fb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #5: + 0xba98cb (0x73a8f50778cb in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #6: + 0xbb1084 (0x73a8f507f084 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_cuda.so) +frame #7: + 0x453c52 (0x73a957e3ac52 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #8: c10::TensorImpl::~TensorImpl() + 0x9 (0x73a8f443f409 in /usr/local/lib/python3.12/dist-packages/torch/lib/libc10.so) +frame #9: + 0x70d688 (0x73a9580f4688 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #10: + 0x70dab0 (0x73a9580f4ab0 in /usr/local/lib/python3.12/dist-packages/torch/lib/libtorch_python.so) +frame #11: /usr/bin/python() [0x5761fe] +frame #12: /usr/bin/python() [0x575f4c] +frame #13: /usr/bin/python() [0x55bae4] +frame #14: /usr/bin/python() [0x579d12] +frame #15: /usr/bin/python() [0x59f0b9] +frame #16: /usr/bin/python() [0x559121] +frame #17: /usr/bin/python() [0x610bf5] +frame #18: /usr/bin/python() [0x610c05] +frame #19: /usr/bin/python() [0x5736c6] +frame #20: PyDict_SetItemString + 0x94 (0x574b14 in /usr/bin/python) +frame #21: /usr/bin/python() [0x6b8165] +frame #22: /usr/bin/python() [0x6b18c2] +frame #23: Py_FinalizeEx + 0xad (0x6b0fad in /usr/bin/python) +frame #24: Py_RunMain + 0x281 (0x6bc961 in /usr/bin/python) +frame #25: Py_BytesMain + 0x2d (0x6bc57d in /usr/bin/python) +frame #26: + 0x2a1ca (0x73a972f021ca in /lib/x86_64-linux-gnu/libc.so.6) +frame #27: __libc_start_main + 0x8b (0x73a972f0228b in /lib/x86_64-linux-gnu/libc.so.6) +frame #28: _start + 0x25 (0x657ce5 in /usr/bin/python) + +W0830 09:42:39.888000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940811 closing signal SIGTERM +W0830 09:42:39.889000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940813 closing signal SIGTERM +W0830 09:42:39.889000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940814 closing signal SIGTERM +W0830 09:42:39.889000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940815 closing signal SIGTERM +W0830 09:42:39.890000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940816 closing signal SIGTERM +W0830 09:42:39.890000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940817 closing signal SIGTERM +W0830 09:42:39.890000 940602 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 940818 closing signal SIGTERM +E0830 09:42:42.462000 940602 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -6) local_rank: 1 (pid: 940812) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-30_09:42:39 + host : sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 + rank : 1 (local_rank: 1) + exitcode : -6 (pid: 940812) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 940812 +============================================================ +Generated: + /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log new file mode 100644 index 0000000000000000000000000000000000000000..c9923d6b43bdf892817792a3447a91123c6c3647 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/verify_trace_ranks.log @@ -0,0 +1,6 @@ +report : /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep +kernels : 141526 +devices : 8 of 8 -> [0, 1, 2, 3, 4, 5, 6, 7] +processes: 1 +markers : 17408 moonep_trace__ kernels on devices [0, 1, 2, 3, 4, 5, 6, 7] +PASS: all ranks present and backend markers emitted diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env new file mode 100644 index 0000000000000000000000000000000000000000..0bf615f8652e8b779bd75420291e406454880472 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=G4 +family=tierA +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce +commit=2e68c8ad +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=16 +hidden=4096 +ffn_hidden=14336 +moe_ffn_hidden=896 +num_experts=128 +topk=32 +experts_per_rank=16 +expert_params=11010048 +heads=32 +kv_groups=8 +kv_channels= +qk_layernorm=0 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=16 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-5 +seq_length=4096 +micro_batch_size=1 +global_batch_size=16 +num_microbatches=2 +assignments_per_rank=131072 +mean_tokens_per_expert=8192 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=1 +probe_mode=0 +async_grad_reduce=1 +branch_role=timing +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=10 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=1 +kernel_trace=1 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log new file mode 100644 index 0000000000000000000000000000000000000000..25f6ca3ad11929840b70d1245034cd97010bd586 --- /dev/null +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/train.log @@ -0,0 +1,1382 @@ +============================================================== + G4 / moonep (slots/rank=16 policy=matched) + E=128 K=32 H'=896 m=16 + L=16 seq=4096 gbs=16 (2 microbatches) + cap_mode=none cap=none cg=1 recompute=none + memory_dump=none snapshots=/workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G4__moonep__matched__seq4096__gbs16__capnone__async/nsys/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 2e68c8ad +============================================================== +W0830 09:43:03.075000 944198 torch/distributed/run.py:766] +W0830 09:43:03.075000 944198 torch/distributed/run.py:766] ***************************************** +W0830 09:43:03.075000 944198 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0830 09:43:03.075000 944198 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. False + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full_iteration + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... True + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 16 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 14336 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 16 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 896 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 32 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 16 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... True + profile_ranks ................................... [0, 1, 2, 3, 4, 5, 6, 7] + profile_step_end ................................ 8 + profile_step_start .............................. 6 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. True + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 10 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +[Gloo] Rank 0 is connected to > initialized pipeline model parallel with size 10 peer ranks. +Expected number of connected peer ranks is : 0 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W830 09:43:17.284436708 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W830 09:43:17.284458431 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W830 09:43:17.289279717 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W830 09:43:17.289305872 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W830 09:43:17.289755909 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W830 09:43:17.290126880 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W830 09:43:17.290233819 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.430 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W830 09:43:17.906751992 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.671 seconds +time to initialize megatron (seconds): 9.364 +[after megatron is initialized] datetime: 2026-08-30 09:43:23 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 3910275072 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=False, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 18 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (42479616 elements, 42479616 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.self_attention.linear_qkv.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_proj.weight +Params for bucket 3 (42475520 elements, 42475520 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_proj.weight +Params for bucket 4 (42475520 elements, 42475520 padded size): + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.13.self_attention.linear_proj.weight +Params for bucket 5 (42475520 elements, 42475520 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight +Params for bucket 6 (42475520 elements, 42475520 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 7 (42475520 elements, 42475520 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 8 (42475520 elements, 42475520 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_qkv.weight +Params for bucket 9 (42475520 elements, 42475520 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 10 (42475520 elements, 42475520 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.linear_qkv.weight +Params for bucket 11 (42475520 elements, 42475520 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.pre_mlp_layernorm.weight +Params for bucket 12 (42475520 elements, 42475520 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.self_attention.linear_qkv.weight +Params for bucket 13 (42475520 elements, 42475520 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight +Params for bucket 14 (42475520 elements, 42475520 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight +Params for bucket 15 (42475520 elements, 42475520 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight +Params for bucket 16 (42475520 elements, 42475520 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight +Params for bucket 17 (42475520 elements, 42475520 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight +Params for bucket 18 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 67 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight9 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc2.weight10 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight13 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 +Params for bucket 3 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 +Params for bucket 4 (44040192 elements, 44040192 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc1.weight10 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight12 + module.decoder.layers.14.mlp.experts.linear_fc2.weight10 + module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight9 + module.decoder.layers.14.mlp.experts.linear_fc2.weight13 + module.decoder.layers.14.mlp.experts.linear_fc2.weight11 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 + module.decoder.layers.14.mlp.experts.linear_fc1.weight11 + module.decoder.layers.14.mlp.experts.linear_fc1.weight13 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight12 + module.decoder.layers.14.mlp.experts.linear_fc1.weight10 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight14 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight11 + module.decoder.layers.13.mlp.experts.linear_fc2.weight13 + module.decoder.layers.13.mlp.experts.linear_fc2.weight15 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight10 + module.decoder.layers.13.mlp.experts.linear_fc2.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight12 + module.decoder.layers.13.mlp.experts.linear_fc2.weight14 +Params for bucket 11 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 +Params for bucket 12 (44040192 elements, 44040192 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight10 + module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc1.weight12 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight9 + module.decoder.layers.13.mlp.experts.linear_fc1.weight11 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight13 + module.decoder.layers.13.mlp.experts.linear_fc1.weight15 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight9 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc2.weight11 + module.decoder.layers.12.mlp.experts.linear_fc2.weight13 + module.decoder.layers.12.mlp.experts.linear_fc2.weight15 +Params for bucket 15 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 +Params for bucket 16 (44040192 elements, 44040192 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight9 + module.decoder.layers.12.mlp.experts.linear_fc1.weight11 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight10 + module.decoder.layers.12.mlp.experts.linear_fc1.weight8 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight13 + module.decoder.layers.12.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 +Params for bucket 23 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 24 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 +Params for bucket 25 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 +Params for bucket 28 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 +Params for bucket 29 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 37 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 42 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 43 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 +Params for bucket 44 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 +Params for bucket 46 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 47 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 +Params for bucket 48 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 49 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 +Params for bucket 50 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 +Params for bucket 51 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 +Params for bucket 52 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 53 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 54 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 +Params for bucket 55 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 56 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 +Params for bucket 59 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 60 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 61 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 +Params for bucket 62 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 +Params for bucket 63 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 +Params for bucket 64 (40370176 elements, 40370176 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +Params for bucket 66 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 67 (29360128 elements, 29360128 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-30 09:43:27 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 160 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(160, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 68684a515e118253a8572c4517cf2ffe-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-30 09:43:27 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3356.97, 3386.96) + train/valid/test-data-iterators-setup ..........: (85.46, 101.70) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-30 09:43:27 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=4096, H=4096, E=128, K=32, NvS=135136, NvS_padded=135168 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 1.03 GiB + meta_buffer_size : 4.00 MiB + local_temp_buffer_size : 4.29 MiB + total_size : 1.04 GiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [896, 22528, 5888, 2816, 10880, 3968, 7424, 7040, 8448, 1664, 9728, 2560, 7552, 5888, 1664, 3456, 23936, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[32, 40, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [103, 107, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [82, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [6400, 256, 22528, 5376, 22912, 128, 1152, 22016, 0, 6656, 4480, 768, 15488, 4480, 128, 19200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [98, 106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [91, 80, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [8, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [100, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [102, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [21632, 128, 26496, 5248, 7168, 24064, 128, 19328, 128, 0, 256, 0, 14592, 6656, 384, 3840, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, 85, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [49, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [9, 11, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [61, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 1280, 16640, 6144, 7680, 1280, 3712, 2048, 22912, 18304, 10496, 0, 8960, 22528, 1664, 128, 8320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[98, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, 93, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [59, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [16, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] + [2026-08-30 09:43:49] iteration 1/ 10 | consumed samples: 16 | elapsed time per iteration (ms): 21840.0 | throughput per GPU (TFLOP/s/GPU): 15.3 | learning rate: 2.000000E-05 | global batch size: 16 | lm loss: 1.102493E+01 | loss scale: 1.0 | grad norm: 44.120 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 23.22 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 23.63 +Number of parameters in most loaded shard in billions: 23.6320 +Theoretical memory footprints: weight and optimizer=169029.43 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 53108.53564453125 | max allocated: 55305.33447265625 | reserved: 57154.0 | max reserved: 57154.0 + [2026-08-30 09:43:50] iteration 2/ 10 | consumed samples: 32 | elapsed time per iteration (ms): 1284.3 | throughput per GPU (TFLOP/s/GPU): 259.6 | learning rate: 4.000000E-05 | global batch size: 16 | lm loss: 1.101859E+01 | loss scale: 1.0 | grad norm: 45.887 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:43:51] iteration 3/ 10 | consumed samples: 48 | elapsed time per iteration (ms): 721.4 | throughput per GPU (TFLOP/s/GPU): 462.1 | learning rate: 6.000000E-05 | global batch size: 16 | lm loss: 1.060162E+01 | loss scale: 1.0 | grad norm: 216.508 | number of skipped iterations: 0 | number of nan iterations: 0 | +Capture CUDA graph for training!!! +CUDA graph capture done!!! + [2026-08-30 09:43:53] iteration 4/ 10 | consumed samples: 64 | elapsed time per iteration (ms): 1792.2 | throughput per GPU (TFLOP/s/GPU): 186.0 | learning rate: 8.000000E-05 | global batch size: 16 | lm loss: 1.301611E+01 | loss scale: 1.0 | grad norm: 46.056 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:43:53] iteration 5/ 10 | consumed samples: 80 | elapsed time per iteration (ms): 785.1 | throughput per GPU (TFLOP/s/GPU): 424.6 | learning rate: 1.000000E-04 | global batch size: 16 | lm loss: 1.033488E+01 | loss scale: 1.0 | grad norm: 17.387 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:43:54] iteration 6/ 10 | consumed samples: 96 | elapsed time per iteration (ms): 787.7 | throughput per GPU (TFLOP/s/GPU): 423.2 | learning rate: 7.750000E-05 | global batch size: 16 | lm loss: 1.016456E+01 | loss scale: 1.0 | grad norm: 17.580 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:43:58] iteration 7/ 10 | consumed samples: 112 | elapsed time per iteration (ms): 3592.9 | throughput per GPU (TFLOP/s/GPU): 92.8 | learning rate: 3.250000E-05 | global batch size: 16 | lm loss: 9.899017E+00 | loss scale: 1.0 | grad norm: 6.187 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-30 09:43:59] iteration 8/ 10 | consumed samples: 128 | elapsed time per iteration (ms): 824.7 | throughput per GPU (TFLOP/s/GPU): 404.2 | learning rate: 1.000000E-05 | global batch size: 16 | lm loss: 9.319439E+00 | loss scale: 1.0 | grad norm: 5.515 | number of skipped iterations: 0 | number of nan iterations: 0 | diff --git a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv index c32a42a518103a13e73118a87b487059eb650427..be2a8245376fc51b040fdbf442d6a272d70c5f99 100644 --- a/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv +++ b/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/trace_results.tsv @@ -1 +1,4 @@ model backend capacity status report async_overlap_json +G1 moonep none OK /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G1__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +G2 moonep none OK /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G2__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json +G3 moonep none OK /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep /workspace/tiera-rerun-fixed-moonep/moonep/tierbtrace/tierbtrace_gbs16/G3__moonep__matched__seq4096__gbs16__capnone__async/nsys/moonep_async_overlap.json diff --git a/tiera-rerun-fixed-moonep/study.log b/tiera-rerun-fixed-moonep/study.log index bf19abd12faccb3c78f5639742877ed86087e69e..b6c09308091f6b4045b83b09d04ead039bdb4368 100644 --- a/tiera-rerun-fixed-moonep/study.log +++ b/tiera-rerun-fixed-moonep/study.log @@ -2057,3 +2057,19 @@ https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [09:32:42] marker extension ready: _moonep_trace_C.cpython-312-x86_64-linux-gnu.so [09:32:42] TRACE-ONLY G1__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7) [guard] iter 6/10 (60s) + [guard] iter 8/10 (70s) + [guard] iter 10/10 (110s) + [guard] nsys stop (flushing report) +[09:35:04] trace verified -> G1__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep (9.3M, all 8 GPUs + moonep markers) +[09:35:05] TRACE-ONLY G2__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7) + [guard] iter 6/10 (60s) + [guard] iter 8/10 (70s) +[09:38:56] WARN: nsys run failed for G2__moonep__matched__seq4096__gbs16__capnone__async (checking report anyway) +[09:38:57] trace verified -> G2__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep (10M, all 8 GPUs + moonep markers) +[09:38:58] TRACE-ONLY G3__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7) + [guard] iter 6/10 (60s) + [guard] iter 8/10 (70s) +[09:42:49] WARN: nsys run failed for G3__moonep__matched__seq4096__gbs16__capnone__async (checking report anyway) +[09:42:50] trace verified -> G3__moonep__matched__seq4096__gbs16__capnone__async.nsys-rep (12M, all 8 GPUs + moonep markers) +[09:42:51] TRACE-ONLY G4__moonep__matched__seq4096__gbs16__capnone__async (ranks: 0 1 2 3 4 5 6 7) + [guard] iter 3/10 (60s)