#pragma once #include #include #include // ----------------------------------------------------------------------------- // CUDA Kernels for Vision Encoder // ----------------------------------------------------------------------------- __global__ void im2col_3x3_kernel( const float* __restrict__ data_im, float* __restrict__ data_col, int channels, int height, int width, int pad, int stride, int out_h, int out_w ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total_spatial = out_h * out_w; int total = channels * total_spatial; if (idx >= total) return; int spatial_idx = idx % total_spatial; int c = idx / total_spatial; int w_out = spatial_idx % out_w; int h_out = spatial_idx / out_w; int h_in_base = h_out * stride - pad; int w_in_base = w_out * stride - pad; const float* im_c = data_im + c * (height * width); int col_channel_base = c * 9; #pragma unroll for (int kh = 0; kh < 3; ++kh) { int h_in = h_in_base + kh; #pragma unroll for (int kw = 0; kw < 3; ++kw) { int w_in = w_in_base + kw; float val = 0.0f; if (h_in >= 0 && h_in < height && w_in >= 0 && w_in < width) { val = im_c[h_in * width + w_in]; } int col_idx = (col_channel_base + kh * 3 + kw) * total_spatial + spatial_idx; data_col[col_idx] = val; } } } __global__ void add_bias_2d_kernel( float* __restrict__ data, const float* __restrict__ bias, int channels, int spatial_dim ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total = channels * spatial_dim; if (idx >= total) return; int c = idx / spatial_dim; data[idx] += bias[c]; } __global__ void silu_kernel(float* data, int total) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx < total) { float x = data[idx]; data[idx] = x / (1.0f + __expf(-x)); } } __global__ void avg_pool_2x2_kernel( const float* __restrict__ in, float* __restrict__ out, int C, int H_out, int W_out ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total = C * H_out * W_out; if (idx >= total) return; int w = idx % W_out; int h = (idx / W_out) % H_out; int c = idx / (H_out * W_out); int in_w = w * 2; int in_h = h * 2; int in_w_total = W_out * 2; int in_h_total = H_out * 2; const float* in_c = in + c * (in_h_total * in_w_total); float val = 0.25f * ( in_c[in_h * in_w_total + in_w] + in_c[in_h * in_w_total + in_w + 1] + in_c[(in_h + 1) * in_w_total + in_w] + in_c[(in_h + 1) * in_w_total + in_w + 1] ); out[idx] = val; } __global__ void gated_residual_kernel( const float* __restrict__ f_base, const float* __restrict__ f_proj, const float* __restrict__ f_gate, float* __restrict__ f_out, int total ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx < total) { float gate = 1.0f / (1.0f + __expf(-f_gate[idx])); f_out[idx] = f_base[idx] + f_proj[idx] * gate; } } __global__ void add_residual_kernel( const float* __restrict__ a, const float* __restrict__ b, float* __restrict__ out, int total ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx < total) { out[idx] = a[idx] + b[idx]; } } // GroupNorm(num_groups=8) + optional SiLU __global__ void groupnorm_2d_kernel( const float* __restrict__ in, const float* __restrict__ gamma, const float* __restrict__ beta, float* __restrict__ out, int C, int H, int W, bool apply_silu ) { // 1 block per group (8 groups total) int group_idx = blockIdx.x; int num_groups = 8; int channels_per_group = C / num_groups; int spatial = H * W; int M = channels_per_group * spatial; int c_start = group_idx * channels_per_group; int tid = threadIdx.x; int bdim = blockDim.x; // Pass 1: compute mean and variance float local_sum = 0.0f; float local_sq_sum = 0.0f; for (int i = tid; i < M; i += bdim) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; float x = in[c * spatial + s]; local_sum += x; local_sq_sum += x * x; } // Warp shuffle reduction for (int offset = 16; offset > 0; offset /= 2) { local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset); } __shared__ float s_sum[32]; __shared__ float s_sq_sum[32]; int lane = tid % 32; int wid = tid / 32; if (lane == 0) { s_sum[wid] = local_sum; s_sq_sum[wid] = local_sq_sum; } __syncthreads(); if (wid == 0) { float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f; for (int offset = 16; offset > 0; offset /= 2) { val += __shfl_down_sync(0xffffffff, val, offset); val_sq += __shfl_down_sync(0xffffffff, val_sq, offset); } if (lane == 0) { s_sum[0] = val; s_sq_sum[0] = val_sq; } } __syncthreads(); float mean = s_sum[0] / (float)M; float var = (s_sq_sum[0] / (float)M) - (mean * mean); if (var < 0.0f) var = 0.0f; float inv_std = rsqrtf(var + 1e-5f); // Pass 2: Normalize and write output for (int i = tid; i < M; i += bdim) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; int idx = c * spatial + s; float x = in[idx]; float norm_x = (x - mean) * inv_std; float y = norm_x * gamma[c] + beta[c]; if (apply_silu) { y = y / (1.0f + __expf(-y)); } out[idx] = y; } } // GroupNorm + FiLM scale/shift + SiLU in a single fused kernel __global__ void groupnorm_film_silu_2d_kernel( const float* __restrict__ in, const float* __restrict__ gamma, const float* __restrict__ beta, const float* __restrict__ scale, const float* __restrict__ shift, float* __restrict__ out, int C, int H, int W ) { int group_idx = blockIdx.x; int num_groups = 8; int channels_per_group = C / num_groups; int spatial = H * W; int M = channels_per_group * spatial; int c_start = group_idx * channels_per_group; int tid = threadIdx.x; int bdim = blockDim.x; float local_sum = 0.0f; float local_sq_sum = 0.0f; for (int i = tid; i < M; i += bdim) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; float x = in[c * spatial + s]; local_sum += x; local_sq_sum += x * x; } for (int offset = 16; offset > 0; offset /= 2) { local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset); } __shared__ float s_sum[32]; __shared__ float s_sq_sum[32]; int lane = tid % 32; int wid = tid / 32; if (lane == 0) { s_sum[wid] = local_sum; s_sq_sum[wid] = local_sq_sum; } __syncthreads(); if (wid == 0) { float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f; for (int offset = 16; offset > 0; offset /= 2) { val += __shfl_down_sync(0xffffffff, val, offset); val_sq += __shfl_down_sync(0xffffffff, val_sq, offset); } if (lane == 0) { s_sum[0] = val; s_sq_sum[0] = val_sq; } } __syncthreads(); float mean = s_sum[0] / (float)M; float var = (s_sq_sum[0] / (float)M) - (mean * mean); if (var < 0.0f) var = 0.0f; float inv_std = rsqrtf(var + 1e-5f); for (int i = tid; i < M; i += bdim) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; int idx = c * spatial + s; float x = in[idx]; float norm_x = (x - mean) * inv_std; // GroupNorm affine float gn_out = norm_x * gamma[c] + beta[c]; // FiLM modulation: gn_out * (1 + scale) + shift float film_out = gn_out * (1.0f + scale[c]) + shift[c]; // SiLU out[idx] = film_out / (1.0f + __expf(-film_out)); } } // ------------------------------------------------------------- // Parallel High-Occupancy GroupNorm (64 blocks: 8 groups x 8 workers) // ------------------------------------------------------------- __global__ void fast_groupnorm_stat_kernel( const float* __restrict__ in, float* __restrict__ group_stats, int C, int spatial ) { int group_idx = blockIdx.x; int block_in_group = blockIdx.y; int num_blocks_per_group = gridDim.y; int channels_per_group = C / 8; int M = channels_per_group * spatial; int c_start = group_idx * channels_per_group; int tid = threadIdx.x; int bdim = blockDim.x; float local_sum = 0.0f; float local_sq_sum = 0.0f; int stride = bdim * num_blocks_per_group; int start = block_in_group * bdim + tid; for (int i = start; i < M; i += stride) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; float x = in[c * spatial + s]; local_sum += x; local_sq_sum += x * x; } for (int offset = 16; offset > 0; offset /= 2) { local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset); } __shared__ float s_sum[32]; __shared__ float s_sq_sum[32]; int lane = tid % 32; int wid = tid / 32; if (lane == 0) { s_sum[wid] = local_sum; s_sq_sum[wid] = local_sq_sum; } __syncthreads(); if (wid == 0) { float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f; for (int offset = 16; offset > 0; offset /= 2) { val += __shfl_down_sync(0xffffffff, val, offset); val_sq += __shfl_down_sync(0xffffffff, val_sq, offset); } if (lane == 0) { atomicAdd(&group_stats[group_idx * 2 + 0], val); atomicAdd(&group_stats[group_idx * 2 + 1], val_sq); } } } __global__ void fast_groupnorm_apply_kernel( const float* __restrict__ in, const float* __restrict__ group_stats, const float* __restrict__ gamma, const float* __restrict__ beta, float* __restrict__ out, int C, int spatial, bool apply_silu ) { int group_idx = blockIdx.x; int block_in_group = blockIdx.y; int num_blocks_per_group = gridDim.y; int channels_per_group = C / 8; int M = channels_per_group * spatial; int c_start = group_idx * channels_per_group; float sum = group_stats[group_idx * 2 + 0]; float sq_sum = group_stats[group_idx * 2 + 1]; float mean = sum / (float)M; float var = (sq_sum / (float)M) - (mean * mean); float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f); int tid = threadIdx.x; int bdim = blockDim.x; int stride = bdim * num_blocks_per_group; int start = block_in_group * bdim + tid; for (int i = start; i < M; i += stride) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; int idx = c * spatial + s; float x = in[idx]; float norm = (x - mean) * inv_std; float y = norm * gamma[c] + beta[c]; if (apply_silu) { y = y * (1.0f / (1.0f + __expf(-y))); } out[idx] = y; } } __global__ void fast_groupnorm_film_silu_apply_kernel( const float* __restrict__ in, const float* __restrict__ group_stats, const float* __restrict__ gamma, const float* __restrict__ beta, const float* __restrict__ scale, const float* __restrict__ shift, float* __restrict__ out, int C, int spatial ) { int group_idx = blockIdx.x; int block_in_group = blockIdx.y; int num_blocks_per_group = gridDim.y; int channels_per_group = C / 8; int M = channels_per_group * spatial; int c_start = group_idx * channels_per_group; float sum = group_stats[group_idx * 2 + 0]; float sq_sum = group_stats[group_idx * 2 + 1]; float mean = sum / (float)M; float var = (sq_sum / (float)M) - (mean * mean); float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f); int tid = threadIdx.x; int bdim = blockDim.x; int stride = bdim * num_blocks_per_group; int start = block_in_group * bdim + tid; for (int i = start; i < M; i += stride) { int c_local = i / spatial; int s = i % spatial; int c = c_start + c_local; int idx = c * spatial + s; float x = in[idx]; float norm_x = (x - mean) * inv_std; float h = norm_x * gamma[c] + beta[c]; float modulated = h * (1.0f + scale[c]) + shift[c]; float out_val = modulated * (1.0f / (1.0f + __expf(-modulated))); out[idx] = out_val; } } // Multi-head attention softmax kernel for AttnPool: 4 heads, 4 queries, N keys __global__ void attn_pool_softmax_kernel( float* __restrict__ scores, int N // number of spatial tokens (e.g. 16384, 4096, 1024, 256) ) { // blockIdx.x: query_idx in [0..3] // blockIdx.y: head_idx in [0..3] int q_idx = blockIdx.x; int h_idx = blockIdx.y; int tid = threadIdx.x; int bdim = blockDim.x; float* row = scores + (h_idx * 4 + q_idx) * N; // Find max for numerical stability float local_max = -1e20f; for (int i = tid; i < N; i += bdim) { local_max = fmaxf(local_max, row[i]); } for (int offset = 16; offset > 0; offset /= 2) { local_max = fmaxf(local_max, __shfl_down_sync(0xffffffff, local_max, offset)); } __shared__ float s_max[32]; int lane = tid % 32; int wid = tid / 32; if (lane == 0) s_max[wid] = local_max; __syncthreads(); if (wid == 0) { float val = (tid < (bdim / 32)) ? s_max[lane] : -1e20f; for (int offset = 16; offset > 0; offset /= 2) { val = fmaxf(val, __shfl_down_sync(0xffffffff, val, offset)); } if (lane == 0) s_max[0] = val; } __syncthreads(); float row_max = s_max[0]; // Compute exp and sum float local_sum = 0.0f; for (int i = tid; i < N; i += bdim) { float e = __expf(row[i] - row_max); row[i] = e; local_sum += e; } for (int offset = 16; offset > 0; offset /= 2) { local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); } __shared__ float s_sum[32]; if (lane == 0) s_sum[wid] = local_sum; __syncthreads(); if (wid == 0) { float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; for (int offset = 16; offset > 0; offset /= 2) { val += __shfl_down_sync(0xffffffff, val, offset); } if (lane == 0) s_sum[0] = val; } __syncthreads(); float inv_sum = 1.0f / (s_sum[0] + 1e-12f); // Normalize for (int i = tid; i < N; i += bdim) { row[i] *= inv_sum; } } // ----------------------------------------------------------------------------- // Vision Encoder Forward Function Declaration // ----------------------------------------------------------------------------- class ParvulusEngine; void dual_view_encoder_forward( ParvulusEngine* engine, cublasHandle_t handle, cudaStream_t stream, const float* d_obs, const float* d_cond, float* d_z_vis, float* d_scratch_a, float* d_scratch_b, float* d_scratch_c, float* d_scratch_d, float* d_im2col, float* d_film_ss, float* d_pool_k, float* d_pool_v, float* d_pool_k_head, float* d_pool_v_head, float* d_pool_q, float* d_pool_q_head, float* d_pool_scores, float* d_pool_ctx_head, float* d_pool_context ); void conv2d_3x3_forward( cublasHandle_t handle, cudaStream_t stream, const float* d_in, const float* d_weight, const float* d_bias, float* d_out, float* d_im2col, int in_c, int in_h, int in_w, int out_c, int stride, int pad );