Upload logs/probepos_logs/h200_build.log with huggingface_hub

logs/probepos_logs/h200_build.log

@@ -0,0 +1,82 @@
+Compiling [CXX  | cp]: Scripts/Blutrain/bluscriptCP.cpp
+Compiling [CXX  | cp]: ../../dist/Data-Parallel/src/DataParallel.cpp
+Compiling [CXX  | cp]: ../../dist/Data-Parallel/src/profiler.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c Scripts/Blutrain/bluscriptCP.cpp -o build/objects/Scripts/Blutrain/bluscriptCP.o
+Compiling [CXX  | cp]: context_parallel/YARNOps.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/Data-Parallel/src/DataParallel.cpp -o build/objects/../../dist/Data-Parallel/src/DataParallel.o
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/Data-Parallel/src/profiler.cpp -o build/objects/../../dist/Data-Parallel/src/profiler.o
+Compiling [CXX  | cp]: process_group/CPProcessGroupNCCL.cpp
+Compiling [CXX  | cp]: context_parallel/ContextParallel.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/YARNOps.cpp -o build/objects/context_parallel/YARNOps.o
+Compiling [CXX  | cp]: process_group/device_mesh.cpp
+Compiling [CXX  | cp]: ../../dist/communication/src/processGroupNccl.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/ContextParallel.cpp -o build/objects/context_parallel/ContextParallel.o
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c process_group/CPProcessGroupNCCL.cpp -o build/objects/process_group/CPProcessGroupNCCL.o
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c process_group/device_mesh.cpp -o build/objects/process_group/device_mesh.o
+Compiling [CXX  | cp]: ../../dist/Data-Parallel/src/Error_logs.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/communication/src/processGroupNccl.cpp -o build/objects/../../dist/communication/src/processGroupNccl.o
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/Data-Parallel/src/Error_logs.cpp -o build/objects/../../dist/Data-Parallel/src/Error_logs.o
+Compiling [CXX  | cp]: ../../dist/zero/src/ShardedOptimizer.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/zero/src/ShardedOptimizer.cpp -o build/objects/../../dist/zero/src/ShardedOptimizer.o
+Compiling [CXX  | cp]: ../../dist/zero/src/SymmetricShardedOptimizer.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/zero/src/SymmetricShardedOptimizer.cpp -o build/objects/../../dist/zero/src/SymmetricShardedOptimizer.o
+Compiling [CXX  | cp]: ../../dist/zero/src/ShardedMuon.cpp
+Compiling [CXX  | cp]: ../../dist/communication/src/NcclSymmetricAllocator.cpp
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/zero/src/ShardedMuon.cpp -o build/objects/../../dist/zero/src/ShardedMuon.o
+Compiling [CUDA | cp]: context_parallel/fused_qknorm_rope_cp_fwd.cu
+Compiling [CUDA | cp]: context_parallel/GQA_fused_bwd_sm103_cp.cu
+Compiling [CUDA | cp]: context_parallel/GQA_fused_fwd_sm103_cp.cu
+Compiling [CUDA | cp]: context_parallel/headtail_kernel.cu
+/usr/bin/mpic++ -std=c++2a -fPIC -O3 -g -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c ../../dist/communication/src/NcclSymmetricAllocator.cpp -o build/objects/../../dist/communication/src/NcclSymmetricAllocator.o
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/fused_qknorm_rope_cp_fwd.cu -o build/objects/context_parallel/fused_qknorm_rope_cp_fwd.o
+Compiling [CUDA | cp]: context_parallel/AttentionForward.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/GQA_fused_bwd_sm103_cp.cu -o build/objects/context_parallel/GQA_fused_bwd_sm103_cp.o
+Compiling [CUDA | cp]: context_parallel/KVPackKernel.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/GQA_fused_fwd_sm103_cp.cu -o build/objects/context_parallel/GQA_fused_fwd_sm103_cp.o
+Compiling [CUDA | cp]: context_parallel/FusedSDPAKernel.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/headtail_kernel.cu -o build/objects/context_parallel/headtail_kernel.o
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/AttentionForward.cu -o build/objects/context_parallel/AttentionForward.o
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/KVPackKernel.cu -o build/objects/context_parallel/KVPackKernel.o
+Compiling [CUDA | cp]: context_parallel/fused_qknorm_rope_cp_bwd.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/FusedSDPAKernel.cu -o build/objects/context_parallel/FusedSDPAKernel.o
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/fused_qknorm_rope_cp_bwd.cu -o build/objects/context_parallel/fused_qknorm_rope_cp_bwd.o
+Compiling [CUDA | cp]: context_parallel/GQA_causal_cp_fwd_sm90.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -gencode arch=compute_90a,code=sm_90a -c context_parallel/GQA_causal_cp_fwd_sm90.cu -o build/objects/context_parallel/GQA_causal_cp_fwd_sm90.o
+Compiling [CUDA | cp]: context_parallel/FusedSDPABackwardKernel.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/FusedSDPABackwardKernel.cu -o build/objects/context_parallel/FusedSDPABackwardKernel.o
+Compiling [CUDA | cp]: context_parallel/GQA_causal_cp_bwd_sm90.cu
+Compiling [CUDA | cp]: context_parallel/AttentionBackward.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -gencode arch=compute_90a,code=sm_90a -c context_parallel/GQA_causal_cp_bwd_sm90.cu -o build/objects/context_parallel/GQA_causal_cp_bwd_sm90.o
+Compiling [CUDA | cp]: context_parallel/EvalNLLKernel.cu
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/AttentionBackward.cu -o build/objects/context_parallel/AttentionBackward.o
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g --use_fast_math --expt-relaxed-constexpr -ccbin=/usr/bin/mpic++ -MMD -MP -I. -IScripts/Blutrain -I../../Tensor-Implementations/include -I../../Profiler/include -I/usr/local/cuda/include -I../../dist/communication/include -I../../dist/Data-Parallel/include -I../.. -I../../dist/zero/include -I../../dist/Distributed_Checkpointing/include -I../../dist/Tensor-Parallelism/dnn -I../../dist/Tensor-Parallelism/tensor -I../../third_party/cufile/Include -DWITH_CUDA -DCP_FUSED_ROPE -DCP_ATTN_SPLIT -c context_parallel/EvalNLLKernel.cu -o build/objects/context_parallel/EvalNLLKernel.o
+context_parallel/FusedSDPABackwardKernel.cu(46): warning #177-D: variable "BWD_TC_WMMA_M" was declared but never referenced
+  static constexpr int BWD_TC_WMMA_M = 16;
+                       ^
+
+Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"
+
+context_parallel/GQA_causal_cp_bwd_sm90.cu(440): warning #177-D: variable "CONS" was declared but never referenced
+      constexpr int CONS = 256;
+                    ^
+          detected during instantiation of "void OwnTensor::cp::cuda::gqacp_bwd_v44::launch_gqa_causal_cp_bwd_v44<Br,Bc,D>(const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const float *, OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, int, int, int, int, int, int, float, __nv_bool) [with Br=64, Bc=64, D=128]" at line 737
+
+Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"
+
+context_parallel/GQA_causal_cp_bwd_sm90.cu(467): warning #177-D: variable "kvBase" was declared but never referenced
+      const long kvBase = ((long)(b * Hkv + hkv) * T_k + k_row0) * D;
+                 ^
+          detected during instantiation of "void OwnTensor::cp::cuda::gqacp_bwd_v44::launch_gqa_causal_cp_bwd_v44<Br,Bc,D>(const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, const float *, OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, OwnTensor::cp::cuda::gqacp_bwd_v44::bf16 *, int, int, int, int, int, int, float, __nv_bool) [with Br=64, Bc=64, D=128]" at line 737
+
+context_parallel/AttentionForward.cu(390): warning #177-D: function "OwnTensor::cp::launch_fwd_kernel" was declared but never referenced
+  static void launch_fwd_kernel(const CPFwdParams& params, int grid_y)
+              ^
+
+Remark: The warnings can be suppressed with "-diag-suppress <warning-number>"
+
+
+--- Linking bluscriptCP (sm_90): build/bluscriptCP_exec
+/usr/local/cuda/bin/nvcc -std=c++17 -Xcompiler="-fPIC" -arch=sm_90 -O3 -g -ccbin=/usr/bin/mpic++ -L/usr/local/cuda/lib64 -L/usr/local/cuda/targets/x86_64-linux/lib -L../../Tensor-Implementations/lib -L../../Profiler/lib -L/usr/lib/x86_64-linux-gnu/openmpi/lib -Xlinker -rpath -Xlinker /usr/lib/x86_64-linux-gnu/openmpi/lib -Xlinker -rpath -Xlinker '$ORIGIN/../../Tensor-Implementations/lib' -Xlinker -rpath -Xlinker '$ORIGIN/../../Profiler/lib' -o build/bluscriptCP_exec \
+        build/objects/Scripts/Blutrain/bluscriptCP.o build/objects/../../dist/Data-Parallel/src/DataParallel.o build/objects/../../dist/Data-Parallel/src/profiler.o build/objects/context_parallel/YARNOps.o build/objects/context_parallel/ContextParallel.o build/objects/process_group/CPProcessGroupNCCL.o build/objects/process_group/device_mesh.o build/objects/../../dist/communication/src/processGroupNccl.o build/objects/../../dist/Data-Parallel/src/Error_logs.o build/objects/../../dist/zero/src/ShardedOptimizer.o build/objects/../../dist/zero/src/SymmetricShardedOptimizer.o build/objects/../../dist/zero/src/ShardedMuon.o build/objects/../../dist/communication/src/NcclSymmetricAllocator.o build/objects/context_parallel/fused_qknorm_rope_cp_fwd.o build/objects/context_parallel/GQA_fused_bwd_sm103_cp.o build/objects/context_parallel/GQA_fused_fwd_sm103_cp.o build/objects/context_parallel/headtail_kernel.o build/objects/context_parallel/AttentionForward.o build/objects/context_parallel/KVPackKernel.o build/objects/context_parallel/FusedSDPAKernel.o build/objects/context_parallel/fused_qknorm_rope_cp_bwd.o build/objects/context_parallel/GQA_causal_cp_fwd_sm90.o build/objects/context_parallel/FusedSDPABackwardKernel.o build/objects/context_parallel/GQA_causal_cp_bwd_sm90.o build/objects/context_parallel/EvalNLLKernel.o build/objects/context_parallel/AttentionBackward.o \
+        -ltensor \
+        -lmpi -lnccl -lcudart -lcublas -lcublasLt -lcuda -lcurand -lgomp -lstdc++ -lnvidia-ml -lz -lpthread -ldl -lprofiler -Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads