#include "vision_encoder.cuh" #include "kernels.cuh" #include "parvulus_engine.h" #include // ----------------------------------------------------------------------------- // Bias & Transposition Kernels // ----------------------------------------------------------------------------- __global__ void add_bias_1d_kernel(float* data, const float* bias, int dim) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx < dim) { data[idx] += bias[idx]; } } __global__ void add_bias_across_tokens_kernel(float* data, const float* bias, int num_tokens, int dim) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total = num_tokens * dim; if (idx < total) { int d = idx % dim; data[idx] += bias[d]; } } __global__ void transpose_interleaved_to_head_kernel( const float* __restrict__ in, float* __restrict__ out, int N, int H, int D ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total = N * H * D; if (idx >= total) return; int d = idx % D; int h = (idx / D) % H; int n = idx / (H * D); int in_idx = idx; int out_idx = h * (N * D) + n * D + d; out[out_idx] = in[in_idx]; } __global__ void transpose_head_to_interleaved_kernel( const float* __restrict__ in, float* __restrict__ out, int N, int H, int D ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total = N * H * D; if (idx >= total) return; int d = idx % D; int n = (idx / D) % N; int h = idx / (N * D); int out_idx = n * (H * D) + h * D + d; out[out_idx] = in[idx]; } // ----------------------------------------------------------------------------- // Convolution Primitives // ----------------------------------------------------------------------------- void conv2d_3x3_forward( cublasHandle_t handle, cudaStream_t stream, const float* d_in, const float* d_weight, const float* d_bias, float* d_out, float* d_im2col, int in_c, int in_h, int in_w, int out_c, int stride, int pad ) { int out_h = (in_h + 2 * pad - 3) / stride + 1; int out_w = (in_w + 2 * pad - 3) / stride + 1; int total_spatial = out_h * out_w; int total_elements = in_c * total_spatial; int threads = 256; int blocks = (total_elements + threads - 1) / threads; im2col_3x3_kernel<<>>( d_in, d_im2col, in_c, in_h, in_w, pad, stride, out_h, out_w ); float alpha = 1.0f, beta = 0.0f; cublasSgemm( handle, CUBLAS_OP_N, CUBLAS_OP_N, total_spatial, out_c, in_c * 9, &alpha, d_im2col, total_spatial, d_weight, in_c * 9, &beta, d_out, total_spatial ); if (d_bias) { int total = out_c * total_spatial; add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>( d_out, d_bias, out_c, total_spatial ); } } void conv2d_1x1_forward( cublasHandle_t handle, cudaStream_t stream, const float* d_in, const float* d_weight, const float* d_bias, float* d_out, int in_c, int spatial, int out_c ) { float alpha = 1.0f, beta = 0.0f; cublasSgemm( handle, CUBLAS_OP_N, CUBLAS_OP_N, spatial, out_c, in_c, &alpha, d_in, spatial, d_weight, in_c, &beta, d_out, spatial ); if (d_bias) { int total = out_c * spatial; add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>( d_out, d_bias, out_c, spatial ); } } // ----------------------------------------------------------------------------- // FiLM ResBlock 2D Forward // ----------------------------------------------------------------------------- void film_resblock_forward( cublasHandle_t handle, cudaStream_t stream, const float* d_in, float* d_out, float* d_scratch1, float* d_scratch2, float* d_im2col, float* d_film_ss, const float* d_cond, const std::string& prefix, ParvulusEngine* engine, int in_c, int out_c, int H, int W ) { int spatial = H * W; // 1. Fast GroupNorm + SiLU on d_in -> d_scratch1 (64-block parallel reduction) float* g_n1 = engine->get_tensor(prefix + "norm1.weight"); float* b_n1 = engine->get_tensor(prefix + "norm1.bias"); cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream); fast_groupnorm_stat_kernel<<>>(d_in, engine->get_gn_stats(), in_c, spatial); fast_groupnorm_apply_kernel<<>>(d_in, engine->get_gn_stats(), g_n1, b_n1, d_scratch1, in_c, spatial, true); // 2. conv1 (3x3, in_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2 float* w_c1 = engine->get_tensor(prefix + "conv1.weight"); float* b_c1 = engine->get_tensor(prefix + "conv1.bias"); conv2d_3x3_forward(handle, stream, d_scratch1, w_c1, b_c1, d_scratch2, d_im2col, in_c, H, W, out_c, 1, 1); // 3. FiLM linear projection: Linear(256, 2 * out_c) // d_film_ss = cond * W^T + b float* w_film = engine->get_tensor(prefix + "film.weight"); float* b_film = engine->get_tensor(prefix + "film.bias"); float alpha = 1.0f, beta = 0.0f; cublasSgemv(handle, CUBLAS_OP_T, 256, 2 * out_c, &alpha, w_film, 256, d_cond, 1, &beta, d_film_ss, 1); add_bias_1d_kernel<<<(2 * out_c + 255) / 256, 256, 0, stream>>>(d_film_ss, b_film, 2 * out_c); // 4. Fast GroupNorm + FiLM modulation + SiLU on d_scratch2 -> d_scratch1 (64-block parallel reduction) float* g_n2 = engine->get_tensor(prefix + "norm2.weight"); float* b_n2 = engine->get_tensor(prefix + "norm2.bias"); float* d_scale = d_film_ss; float* d_shift = d_film_ss + out_c; cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream); fast_groupnorm_stat_kernel<<>>(d_scratch2, engine->get_gn_stats(), out_c, spatial); fast_groupnorm_film_silu_apply_kernel<<>>( d_scratch2, engine->get_gn_stats(), g_n2, b_n2, d_scale, d_shift, d_scratch1, out_c, spatial ); // 5. conv2 (3x3, out_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2 float* w_c2 = engine->get_tensor(prefix + "conv2.weight"); float* b_c2 = engine->get_tensor(prefix + "conv2.bias"); conv2d_3x3_forward(handle, stream, d_scratch1, w_c2, b_c2, d_scratch2, d_im2col, out_c, H, W, out_c, 1, 1); // 6. Skip connection if (in_c != out_c) { float* w_skip = engine->get_tensor(prefix + "skip.weight"); float* b_skip = engine->get_tensor(prefix + "skip.bias"); conv2d_1x1_forward(handle, stream, d_in, w_skip, b_skip, d_scratch1, in_c, spatial, out_c); add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>( d_scratch2, d_scratch1, d_out, out_c * spatial ); } else { add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>( d_scratch2, d_in, d_out, out_c * spatial ); } } // ------------------------------------------------------------- // Cross-View Mixer Forward // ------------------------------------------------------------- void cv_mixer_forward( cublasHandle_t handle, cudaStream_t stream, const float* d_agent, const float* d_wrist, float* d_agent_out, float* d_wrist_out, float* d_mixed_out, float* d_scratch1, float* d_scratch2, float* d_im2col, const std::string& prefix, ParvulusEngine* engine, int ch, int H, int W ) { int spatial = H * W; // 1. Joint = cat(d_agent, d_wrist) -> shape [2*ch, spatial] in d_mixed_out cudaMemcpyAsync(d_mixed_out, d_agent, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); // 2. context_conv: Conv1x1 (2*ch -> ch) on joint -> GroupNorm(8, ch) -> SiLU -> ctx float* w_ctx = engine->get_tensor(prefix + "context_conv.0.weight"); float* b_ctx = engine->get_tensor(prefix + "context_conv.0.bias"); conv2d_1x1_forward(handle, stream, d_mixed_out, w_ctx, b_ctx, d_scratch1, 2 * ch, spatial, ch); float* g_ctx = engine->get_tensor(prefix + "context_conv.1.weight"); float* b_ctx_gn = engine->get_tensor(prefix + "context_conv.1.bias"); float* d_ctx = d_scratch2; cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream); fast_groupnorm_stat_kernel<<>>(d_scratch1, engine->get_gn_stats(), ch, spatial); fast_groupnorm_apply_kernel<<>>(d_scratch1, engine->get_gn_stats(), g_ctx, b_ctx_gn, d_ctx, ch, spatial, true); // 3. agent_proj (3x3 on ctx) -> d_scratch1 float* w_ap = engine->get_tensor(prefix + "agent_proj.weight"); float* b_ap = engine->get_tensor(prefix + "agent_proj.bias"); conv2d_3x3_forward(handle, stream, d_ctx, w_ap, b_ap, d_scratch1, d_im2col, ch, H, W, ch, 1, 1); // 4. agent_gate (1x1 on ctx) -> d_agent_out (temporary gate buffer) float* w_ag = engine->get_tensor(prefix + "agent_gate.weight"); float* b_ag = engine->get_tensor(prefix + "agent_gate.bias"); conv2d_1x1_forward(handle, stream, d_ctx, w_ag, b_ag, d_agent_out, ch, spatial, ch); // f_agent_out = d_agent + d_scratch1 * sigmoid(d_agent_out) gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>( d_agent, d_scratch1, d_agent_out, d_agent_out, ch * spatial ); // 5. wrist_proj (3x3 on ctx) -> d_scratch1 float* w_wp = engine->get_tensor(prefix + "wrist_proj.weight"); float* b_wp = engine->get_tensor(prefix + "wrist_proj.bias"); conv2d_3x3_forward(handle, stream, d_ctx, w_wp, b_wp, d_scratch1, d_im2col, ch, H, W, ch, 1, 1); // 6. wrist_gate (1x1 on ctx) -> d_wrist_out (temporary gate buffer) float* w_wg = engine->get_tensor(prefix + "wrist_gate.weight"); float* b_wg = engine->get_tensor(prefix + "wrist_gate.bias"); conv2d_1x1_forward(handle, stream, d_ctx, w_wg, b_wg, d_wrist_out, ch, spatial, ch); // f_wrist_out = d_wrist + d_scratch1 * sigmoid(d_wrist_out) gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>( d_wrist, d_scratch1, d_wrist_out, d_wrist_out, ch * spatial ); // 7. f_mixed = cat(f_agent_out, f_wrist_out) cudaMemcpyAsync(d_mixed_out, d_agent_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); } // ------------------------------------------------------------- // Multi-Head Attention Pooling Forward (Fused Projections & Precomputed Q) // ------------------------------------------------------------- void attn_pool_forward( cublasHandle_t handle, cudaStream_t stream, const float* d_mixed, float* d_out_tokens_4x256, float* d_k_proj, // [N, 256] float* d_v_proj, // [N, 256] float* d_k_head, // [4, N, 64] float* d_v_head, // [4, N, 64] float* d_scores, // [4, 4, N] float* d_ctx_head, // [4, 4, 64] float* d_context, // [4, 256] int stage, ParvulusEngine* engine, int in_c, int H, int W ) { int N = H * W; float alpha = 1.0f, beta = 0.0f; // 1. Fused K projection directly from d_mixed (eliminates in_proj GEMM & bias) float* w_k_fused = engine->get_attn_wk_fused(stage); float* b_k_fused = engine->get_attn_bk_fused(stage); cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_k_fused, in_c, d_mixed, N, &beta, d_k_proj, 256); add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_k_proj, b_k_fused, N, 256); transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>( d_k_proj, d_k_head, N, 4, 64 ); // 2. Fused V projection directly from d_mixed (eliminates in_proj GEMM & bias) float* w_v_fused = engine->get_attn_wv_fused(stage); float* b_v_fused = engine->get_attn_bv_fused(stage); cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_v_fused, in_c, d_mixed, N, &beta, d_v_proj, 256); add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_v_proj, b_v_fused, N, 256); transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>( d_v_proj, d_v_head, N, 4, 64 ); // 3. Precomputed Q head (Zero FLOPs, Zero runtime GEMM) float* d_q_head = engine->get_attn_q_head(stage); // 4. Batched GEMM: Scores = Q * K^T * 0.125 float scale = 0.125f; cublasSgemmStridedBatched( handle, CUBLAS_OP_T, CUBLAS_OP_N, N, 4, 64, &scale, d_k_head, 64, 64 * N, d_q_head, 64, 4 * 64, &beta, d_scores, N, 4 * N, 4 ); // 5. Softmax over N keys for each head & query attn_pool_softmax_kernel<<>>(d_scores, N); // 6. Context_h = Scores_h * V_h // Scores_h: (N x 4 col-major) // V_h: (64 x N col-major, lda = 64) // Context_h = V_h * Scores_h: (64 x N) x (N x 4) = (64 x 4 col-major, ldc = 64) cublasSgemmStridedBatched( handle, CUBLAS_OP_N, CUBLAS_OP_N, 64, 4, N, &alpha, d_v_head, 64, 64 * N, d_scores, N, 4 * N, &beta, d_ctx_head, 64, 4 * 64, 4 ); // Transpose context from head-major [4 heads, 4 queries, 64 dims] to interleaved [4 queries, 256 dims] transpose_head_to_interleaved_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream>>>( d_ctx_head, d_context, 4, 4, 64 ); // 7. Out projection: std::string prefix = "encoder.attn_poolers." + std::to_string(stage) + "."; float* w_out = engine->get_tensor(prefix + "cross_attn.out_proj.weight"); float* b_out = engine->get_tensor(prefix + "cross_attn.out_proj.bias"); cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_out, 256, d_context, 256, &beta, d_out_tokens_4x256, 256); add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream>>>(d_out_tokens_4x256, b_out, 4, 256); // 8. LayerNorm(256) float* g_norm = engine->get_tensor(prefix + "norm.weight"); float* b_norm = engine->get_tensor(prefix + "norm.bias"); layernorm_256_kernel<<<4, 256, 0, stream>>>(d_out_tokens_4x256, g_norm, b_norm, d_out_tokens_4x256, 4); } // ----------------------------------------------------------------------------- // Complete DualView Vision Encoder Forward Pass // ----------------------------------------------------------------------------- void dual_view_encoder_forward( ParvulusEngine* engine, cublasHandle_t handle, cudaStream_t stream, const float* d_obs, const float* d_cond, float* d_z_vis, float* d_scratch_a, float* d_scratch_b, float* d_scratch_c, float* d_scratch_d, float* d_im2col, float* d_film_ss, float* d_pool_k, float* d_pool_v, float* d_pool_k_head, float* d_pool_v_head, float* d_pool_q, float* d_pool_q_head, float* d_pool_scores, float* d_pool_ctx_head, float* d_pool_context ) { const float* d_agent_raw = d_obs; const float* d_wrist_raw = d_obs + 3 * 256 * 256; // 1. Stem convolutions: Conv3x3 (3 -> 48, pad=1, stride=1) on 256x256 float* w_a_stem = engine->get_tensor("encoder.agent_stem.weight"); float* b_a_stem = engine->get_tensor("encoder.agent_stem.bias"); conv2d_3x3_forward(handle, stream, d_agent_raw, w_a_stem, b_a_stem, d_scratch_a, d_im2col, 3, 256, 256, 48, 1, 1); float* w_w_stem = engine->get_tensor("encoder.wrist_stem.weight"); float* b_w_stem = engine->get_tensor("encoder.wrist_stem.bias"); conv2d_3x3_forward(handle, stream, d_wrist_raw, w_w_stem, b_w_stem, d_scratch_b, d_im2col, 3, 256, 256, 48, 1, 1); // 2. Downsample stem 0.5x (256x256 -> 128x128): avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_a, d_scratch_c, 48, 128, 128); avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_b, d_scratch_d, 48, 128, 128); int channel_mults[4] = {1, 2, 2, 4}; int in_channels[4] = {48, 48, 96, 96}; int resolutions[4] = {128, 64, 32, 16}; for (int stage = 0; stage < 4; ++stage) { int in_ch = in_channels[stage]; int out_ch = 48 * channel_mults[stage]; int H = resolutions[stage]; int W = resolutions[stage]; std::string pfx_a = "encoder.agent_stages." + std::to_string(stage) + ".0."; std::string pfx_w = "encoder.wrist_stages." + std::to_string(stage) + ".0."; std::string pfx_mix = "encoder.cv_mixers." + std::to_string(stage) + "."; std::string pfx_pool = "encoder.attn_poolers." + std::to_string(stage) + "."; std::string pfx_down = "encoder.downsamples." + std::to_string(stage) + "."; float* d_stage_tokens = d_z_vis + stage * 4 * 256; if (stage % 2 == 0) { // Inputs are in d_scratch_c (agent) and d_scratch_d (wrist) // (a) FiLM ResBlocks -> write to d_scratch_a and d_scratch_b film_resblock_forward( handle, stream, d_scratch_c, d_scratch_a, d_pool_k, d_pool_v, d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W ); film_resblock_forward( handle, stream, d_scratch_d, d_scratch_b, d_pool_k, d_pool_v, d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W ); // (b) CVMixer: inputs (a, b) -> agent_out (c), wrist_out (d), mixed_out (a) cv_mixer_forward( handle, stream, d_scratch_a, d_scratch_b, d_scratch_c, d_scratch_d, d_scratch_a, d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W ); // (c) AttnPool2d: mixed is in d_scratch_a attn_pool_forward( handle, stream, d_scratch_a, d_stage_tokens, d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head, d_pool_scores, d_pool_ctx_head, d_pool_context, stage, engine, 2 * out_ch, H, W ); // (d) Downsample: read (c, d) -> write to (a, b) if (stage < 3) { float* w_down = engine->get_tensor(pfx_down + "weight"); float* b_down = engine->get_tensor(pfx_down + "bias"); conv2d_3x3_forward(handle, stream, d_scratch_c, w_down, b_down, d_scratch_a, d_im2col, out_ch, H, W, out_ch, 2, 1); conv2d_3x3_forward(handle, stream, d_scratch_d, w_down, b_down, d_scratch_b, d_im2col, out_ch, H, W, out_ch, 2, 1); } } else { // Inputs are in d_scratch_a (agent) and d_scratch_b (wrist) // (a) FiLM ResBlocks -> write to d_scratch_c and d_scratch_d film_resblock_forward( handle, stream, d_scratch_a, d_scratch_c, d_pool_k, d_pool_v, d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W ); film_resblock_forward( handle, stream, d_scratch_b, d_scratch_d, d_pool_k, d_pool_v, d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W ); // (b) CVMixer: inputs (c, d) -> agent_out (a), wrist_out (b), mixed_out (c) cv_mixer_forward( handle, stream, d_scratch_c, d_scratch_d, d_scratch_a, d_scratch_b, d_scratch_c, d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W ); // (c) AttnPool2d: mixed is in d_scratch_c attn_pool_forward( handle, stream, d_scratch_c, d_stage_tokens, d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head, d_pool_scores, d_pool_ctx_head, d_pool_context, stage, engine, 2 * out_ch, H, W ); // (d) Downsample: read (a, b) -> write to (c, d) if (stage < 3) { float* w_down = engine->get_tensor(pfx_down + "weight"); float* b_down = engine->get_tensor(pfx_down + "bias"); conv2d_3x3_forward(handle, stream, d_scratch_a, w_down, b_down, d_scratch_c, d_im2col, out_ch, H, W, out_ch, 2, 1); conv2d_3x3_forward(handle, stream, d_scratch_b, w_down, b_down, d_scratch_d, d_im2col, out_ch, H, W, out_ch, 2, 1); } } } } void test_groupnorm( const float* d_in, const float* gamma, const float* beta, float* d_out, int C, int H, int W, bool silu, cudaStream_t stream ) { groupnorm_2d_kernel<<<8, 256, 0, stream>>>(d_in, gamma, beta, d_out, C, H, W, silu); } void test_film_resblock( ParvulusEngine* engine, cublasHandle_t handle, cudaStream_t stream, const float* d_in, float* d_out, float* d_scratch1, float* d_scratch2, float* d_im2col, float* d_film_ss, const float* d_cond, const std::string& prefix, int in_c, int out_c, int H, int W ) { film_resblock_forward( handle, stream, d_in, d_out, d_scratch1, d_scratch2, d_im2col, d_film_ss, d_cond, prefix, engine, in_c, out_c, H, W ); } void test_attn_pool( ParvulusEngine* engine, cublasHandle_t handle, cudaStream_t stream, const float* d_mixed, float* d_out_tokens, float* d_scratch_b, float* d_pool_k, float* d_pool_v, float* d_pool_k_head, float* d_pool_v_head, float* d_pool_q, float* d_pool_q_head, float* d_pool_scores, float* d_pool_ctx_head, float* d_pool_context, const std::string& prefix, int in_c, int H, int W ) { attn_pool_forward( handle, stream, d_mixed, d_out_tokens, d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head, d_pool_scores, d_pool_ctx_head, d_pool_context, 0, engine, in_c, H, W ); }