Download c_engine/vision_encoder.cuh from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 16.3 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/vision_encoder.cuh
- Command line
-
hf download hf://aasivas/Parvulus/c_engine/vision_encoder.cuh
-
curl -L -o vision_encoder.cuh https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/vision_encoder.cuh
16.3 kB
| // ----------------------------------------------------------------------------- | |
| // CUDA Kernels for Vision Encoder | |
| // ----------------------------------------------------------------------------- | |
| __global__ void im2col_3x3_kernel( | |
| const float* __restrict__ data_im, | |
| float* __restrict__ data_col, | |
| int channels, int height, int width, | |
| int pad, int stride, | |
| int out_h, int out_w | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| int total_spatial = out_h * out_w; | |
| int total = channels * total_spatial; | |
| if (idx >= total) return; | |
| int spatial_idx = idx % total_spatial; | |
| int c = idx / total_spatial; | |
| int w_out = spatial_idx % out_w; | |
| int h_out = spatial_idx / out_w; | |
| int h_in_base = h_out * stride - pad; | |
| int w_in_base = w_out * stride - pad; | |
| const float* im_c = data_im + c * (height * width); | |
| int col_channel_base = c * 9; | |
| for (int kh = 0; kh < 3; ++kh) { | |
| int h_in = h_in_base + kh; | |
| for (int kw = 0; kw < 3; ++kw) { | |
| int w_in = w_in_base + kw; | |
| float val = 0.0f; | |
| if (h_in >= 0 && h_in < height && w_in >= 0 && w_in < width) { | |
| val = im_c[h_in * width + w_in]; | |
| } | |
| int col_idx = (col_channel_base + kh * 3 + kw) * total_spatial + spatial_idx; | |
| data_col[col_idx] = val; | |
| } | |
| } | |
| } | |
| __global__ void add_bias_2d_kernel( | |
| float* __restrict__ data, | |
| const float* __restrict__ bias, | |
| int channels, int spatial_dim | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| int total = channels * spatial_dim; | |
| if (idx >= total) return; | |
| int c = idx / spatial_dim; | |
| data[idx] += bias[c]; | |
| } | |
| __global__ void silu_kernel(float* data, int total) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| if (idx < total) { | |
| float x = data[idx]; | |
| data[idx] = x / (1.0f + __expf(-x)); | |
| } | |
| } | |
| __global__ void avg_pool_2x2_kernel( | |
| const float* __restrict__ in, | |
| float* __restrict__ out, | |
| int C, int H_out, int W_out | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| int total = C * H_out * W_out; | |
| if (idx >= total) return; | |
| int w = idx % W_out; | |
| int h = (idx / W_out) % H_out; | |
| int c = idx / (H_out * W_out); | |
| int in_w = w * 2; | |
| int in_h = h * 2; | |
| int in_w_total = W_out * 2; | |
| int in_h_total = H_out * 2; | |
| const float* in_c = in + c * (in_h_total * in_w_total); | |
| float val = 0.25f * ( | |
| in_c[in_h * in_w_total + in_w] + | |
| in_c[in_h * in_w_total + in_w + 1] + | |
| in_c[(in_h + 1) * in_w_total + in_w] + | |
| in_c[(in_h + 1) * in_w_total + in_w + 1] | |
| ); | |
| out[idx] = val; | |
| } | |
| __global__ void gated_residual_kernel( | |
| const float* __restrict__ f_base, | |
| const float* __restrict__ f_proj, | |
| const float* __restrict__ f_gate, | |
| float* __restrict__ f_out, | |
| int total | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| if (idx < total) { | |
| float gate = 1.0f / (1.0f + __expf(-f_gate[idx])); | |
| f_out[idx] = f_base[idx] + f_proj[idx] * gate; | |
| } | |
| } | |
| __global__ void add_residual_kernel( | |
| const float* __restrict__ a, | |
| const float* __restrict__ b, | |
| float* __restrict__ out, | |
| int total | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| if (idx < total) { | |
| out[idx] = a[idx] + b[idx]; | |
| } | |
| } | |
| // GroupNorm(num_groups=8) + optional SiLU | |
| __global__ void groupnorm_2d_kernel( | |
| const float* __restrict__ in, | |
| const float* __restrict__ gamma, | |
| const float* __restrict__ beta, | |
| float* __restrict__ out, | |
| int C, int H, int W, | |
| bool apply_silu | |
| ) { | |
| // 1 block per group (8 groups total) | |
| int group_idx = blockIdx.x; | |
| int num_groups = 8; | |
| int channels_per_group = C / num_groups; | |
| int spatial = H * W; | |
| int M = channels_per_group * spatial; | |
| int c_start = group_idx * channels_per_group; | |
| int tid = threadIdx.x; | |
| int bdim = blockDim.x; | |
| // Pass 1: compute mean and variance | |
| float local_sum = 0.0f; | |
| float local_sq_sum = 0.0f; | |
| for (int i = tid; i < M; i += bdim) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| float x = in[c * spatial + s]; | |
| local_sum += x; | |
| local_sq_sum += x * x; | |
| } | |
| // Warp shuffle reduction | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); | |
| local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset); | |
| } | |
| __shared__ float s_sum[32]; | |
| __shared__ float s_sq_sum[32]; | |
| int lane = tid % 32; | |
| int wid = tid / 32; | |
| if (lane == 0) { | |
| s_sum[wid] = local_sum; | |
| s_sq_sum[wid] = local_sq_sum; | |
| } | |
| __syncthreads(); | |
| if (wid == 0) { | |
| float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; | |
| float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f; | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| val += __shfl_down_sync(0xffffffff, val, offset); | |
| val_sq += __shfl_down_sync(0xffffffff, val_sq, offset); | |
| } | |
| if (lane == 0) { | |
| s_sum[0] = val; | |
| s_sq_sum[0] = val_sq; | |
| } | |
| } | |
| __syncthreads(); | |
| float mean = s_sum[0] / (float)M; | |
| float var = (s_sq_sum[0] / (float)M) - (mean * mean); | |
| if (var < 0.0f) var = 0.0f; | |
| float inv_std = rsqrtf(var + 1e-5f); | |
| // Pass 2: Normalize and write output | |
| for (int i = tid; i < M; i += bdim) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| int idx = c * spatial + s; | |
| float x = in[idx]; | |
| float norm_x = (x - mean) * inv_std; | |
| float y = norm_x * gamma[c] + beta[c]; | |
| if (apply_silu) { | |
| y = y / (1.0f + __expf(-y)); | |
| } | |
| out[idx] = y; | |
| } | |
| } | |
| // GroupNorm + FiLM scale/shift + SiLU in a single fused kernel | |
| __global__ void groupnorm_film_silu_2d_kernel( | |
| const float* __restrict__ in, | |
| const float* __restrict__ gamma, | |
| const float* __restrict__ beta, | |
| const float* __restrict__ scale, | |
| const float* __restrict__ shift, | |
| float* __restrict__ out, | |
| int C, int H, int W | |
| ) { | |
| int group_idx = blockIdx.x; | |
| int num_groups = 8; | |
| int channels_per_group = C / num_groups; | |
| int spatial = H * W; | |
| int M = channels_per_group * spatial; | |
| int c_start = group_idx * channels_per_group; | |
| int tid = threadIdx.x; | |
| int bdim = blockDim.x; | |
| float local_sum = 0.0f; | |
| float local_sq_sum = 0.0f; | |
| for (int i = tid; i < M; i += bdim) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| float x = in[c * spatial + s]; | |
| local_sum += x; | |
| local_sq_sum += x * x; | |
| } | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); | |
| local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset); | |
| } | |
| __shared__ float s_sum[32]; | |
| __shared__ float s_sq_sum[32]; | |
| int lane = tid % 32; | |
| int wid = tid / 32; | |
| if (lane == 0) { | |
| s_sum[wid] = local_sum; | |
| s_sq_sum[wid] = local_sq_sum; | |
| } | |
| __syncthreads(); | |
| if (wid == 0) { | |
| float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; | |
| float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f; | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| val += __shfl_down_sync(0xffffffff, val, offset); | |
| val_sq += __shfl_down_sync(0xffffffff, val_sq, offset); | |
| } | |
| if (lane == 0) { | |
| s_sum[0] = val; | |
| s_sq_sum[0] = val_sq; | |
| } | |
| } | |
| __syncthreads(); | |
| float mean = s_sum[0] / (float)M; | |
| float var = (s_sq_sum[0] / (float)M) - (mean * mean); | |
| if (var < 0.0f) var = 0.0f; | |
| float inv_std = rsqrtf(var + 1e-5f); | |
| for (int i = tid; i < M; i += bdim) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| int idx = c * spatial + s; | |
| float x = in[idx]; | |
| float norm_x = (x - mean) * inv_std; | |
| // GroupNorm affine | |
| float gn_out = norm_x * gamma[c] + beta[c]; | |
| // FiLM modulation: gn_out * (1 + scale) + shift | |
| float film_out = gn_out * (1.0f + scale[c]) + shift[c]; | |
| // SiLU | |
| out[idx] = film_out / (1.0f + __expf(-film_out)); | |
| } | |
| } | |
| // ------------------------------------------------------------- | |
| // Parallel High-Occupancy GroupNorm (64 blocks: 8 groups x 8 workers) | |
| // ------------------------------------------------------------- | |
| __global__ void fast_groupnorm_stat_kernel( | |
| const float* __restrict__ in, | |
| float* __restrict__ group_stats, | |
| int C, int spatial | |
| ) { | |
| int group_idx = blockIdx.x; | |
| int block_in_group = blockIdx.y; | |
| int num_blocks_per_group = gridDim.y; | |
| int channels_per_group = C / 8; | |
| int M = channels_per_group * spatial; | |
| int c_start = group_idx * channels_per_group; | |
| int tid = threadIdx.x; | |
| int bdim = blockDim.x; | |
| float local_sum = 0.0f; | |
| float local_sq_sum = 0.0f; | |
| int stride = bdim * num_blocks_per_group; | |
| int start = block_in_group * bdim + tid; | |
| for (int i = start; i < M; i += stride) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| float x = in[c * spatial + s]; | |
| local_sum += x; | |
| local_sq_sum += x * x; | |
| } | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); | |
| local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset); | |
| } | |
| __shared__ float s_sum[32]; | |
| __shared__ float s_sq_sum[32]; | |
| int lane = tid % 32; | |
| int wid = tid / 32; | |
| if (lane == 0) { | |
| s_sum[wid] = local_sum; | |
| s_sq_sum[wid] = local_sq_sum; | |
| } | |
| __syncthreads(); | |
| if (wid == 0) { | |
| float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; | |
| float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f; | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| val += __shfl_down_sync(0xffffffff, val, offset); | |
| val_sq += __shfl_down_sync(0xffffffff, val_sq, offset); | |
| } | |
| if (lane == 0) { | |
| atomicAdd(&group_stats[group_idx * 2 + 0], val); | |
| atomicAdd(&group_stats[group_idx * 2 + 1], val_sq); | |
| } | |
| } | |
| } | |
| __global__ void fast_groupnorm_apply_kernel( | |
| const float* __restrict__ in, | |
| const float* __restrict__ group_stats, | |
| const float* __restrict__ gamma, | |
| const float* __restrict__ beta, | |
| float* __restrict__ out, | |
| int C, int spatial, | |
| bool apply_silu | |
| ) { | |
| int group_idx = blockIdx.x; | |
| int block_in_group = blockIdx.y; | |
| int num_blocks_per_group = gridDim.y; | |
| int channels_per_group = C / 8; | |
| int M = channels_per_group * spatial; | |
| int c_start = group_idx * channels_per_group; | |
| float sum = group_stats[group_idx * 2 + 0]; | |
| float sq_sum = group_stats[group_idx * 2 + 1]; | |
| float mean = sum / (float)M; | |
| float var = (sq_sum / (float)M) - (mean * mean); | |
| float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f); | |
| int tid = threadIdx.x; | |
| int bdim = blockDim.x; | |
| int stride = bdim * num_blocks_per_group; | |
| int start = block_in_group * bdim + tid; | |
| for (int i = start; i < M; i += stride) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| int idx = c * spatial + s; | |
| float x = in[idx]; | |
| float norm = (x - mean) * inv_std; | |
| float y = norm * gamma[c] + beta[c]; | |
| if (apply_silu) { | |
| y = y * (1.0f / (1.0f + __expf(-y))); | |
| } | |
| out[idx] = y; | |
| } | |
| } | |
| __global__ void fast_groupnorm_film_silu_apply_kernel( | |
| const float* __restrict__ in, | |
| const float* __restrict__ group_stats, | |
| const float* __restrict__ gamma, | |
| const float* __restrict__ beta, | |
| const float* __restrict__ scale, | |
| const float* __restrict__ shift, | |
| float* __restrict__ out, | |
| int C, int spatial | |
| ) { | |
| int group_idx = blockIdx.x; | |
| int block_in_group = blockIdx.y; | |
| int num_blocks_per_group = gridDim.y; | |
| int channels_per_group = C / 8; | |
| int M = channels_per_group * spatial; | |
| int c_start = group_idx * channels_per_group; | |
| float sum = group_stats[group_idx * 2 + 0]; | |
| float sq_sum = group_stats[group_idx * 2 + 1]; | |
| float mean = sum / (float)M; | |
| float var = (sq_sum / (float)M) - (mean * mean); | |
| float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f); | |
| int tid = threadIdx.x; | |
| int bdim = blockDim.x; | |
| int stride = bdim * num_blocks_per_group; | |
| int start = block_in_group * bdim + tid; | |
| for (int i = start; i < M; i += stride) { | |
| int c_local = i / spatial; | |
| int s = i % spatial; | |
| int c = c_start + c_local; | |
| int idx = c * spatial + s; | |
| float x = in[idx]; | |
| float norm_x = (x - mean) * inv_std; | |
| float h = norm_x * gamma[c] + beta[c]; | |
| float modulated = h * (1.0f + scale[c]) + shift[c]; | |
| float out_val = modulated * (1.0f / (1.0f + __expf(-modulated))); | |
| out[idx] = out_val; | |
| } | |
| } | |
| // Multi-head attention softmax kernel for AttnPool: 4 heads, 4 queries, N keys | |
| __global__ void attn_pool_softmax_kernel( | |
| float* __restrict__ scores, | |
| int N // number of spatial tokens (e.g. 16384, 4096, 1024, 256) | |
| ) { | |
| // blockIdx.x: query_idx in [0..3] | |
| // blockIdx.y: head_idx in [0..3] | |
| int q_idx = blockIdx.x; | |
| int h_idx = blockIdx.y; | |
| int tid = threadIdx.x; | |
| int bdim = blockDim.x; | |
| float* row = scores + (h_idx * 4 + q_idx) * N; | |
| // Find max for numerical stability | |
| float local_max = -1e20f; | |
| for (int i = tid; i < N; i += bdim) { | |
| local_max = fmaxf(local_max, row[i]); | |
| } | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| local_max = fmaxf(local_max, __shfl_down_sync(0xffffffff, local_max, offset)); | |
| } | |
| __shared__ float s_max[32]; | |
| int lane = tid % 32; | |
| int wid = tid / 32; | |
| if (lane == 0) s_max[wid] = local_max; | |
| __syncthreads(); | |
| if (wid == 0) { | |
| float val = (tid < (bdim / 32)) ? s_max[lane] : -1e20f; | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| val = fmaxf(val, __shfl_down_sync(0xffffffff, val, offset)); | |
| } | |
| if (lane == 0) s_max[0] = val; | |
| } | |
| __syncthreads(); | |
| float row_max = s_max[0]; | |
| // Compute exp and sum | |
| float local_sum = 0.0f; | |
| for (int i = tid; i < N; i += bdim) { | |
| float e = __expf(row[i] - row_max); | |
| row[i] = e; | |
| local_sum += e; | |
| } | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| local_sum += __shfl_down_sync(0xffffffff, local_sum, offset); | |
| } | |
| __shared__ float s_sum[32]; | |
| if (lane == 0) s_sum[wid] = local_sum; | |
| __syncthreads(); | |
| if (wid == 0) { | |
| float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f; | |
| for (int offset = 16; offset > 0; offset /= 2) { | |
| val += __shfl_down_sync(0xffffffff, val, offset); | |
| } | |
| if (lane == 0) s_sum[0] = val; | |
| } | |
| __syncthreads(); | |
| float inv_sum = 1.0f / (s_sum[0] + 1e-12f); | |
| // Normalize | |
| for (int i = tid; i < N; i += bdim) { | |
| row[i] *= inv_sum; | |
| } | |
| } | |
| // ----------------------------------------------------------------------------- | |
| // Vision Encoder Forward Function Declaration | |
| // ----------------------------------------------------------------------------- | |
| class ParvulusEngine; | |
| void dual_view_encoder_forward( | |
| ParvulusEngine* engine, | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_obs, | |
| const float* d_cond, | |
| float* d_z_vis, | |
| float* d_scratch_a, | |
| float* d_scratch_b, | |
| float* d_scratch_c, | |
| float* d_scratch_d, | |
| float* d_im2col, | |
| float* d_film_ss, | |
| float* d_pool_k, | |
| float* d_pool_v, | |
| float* d_pool_k_head, | |
| float* d_pool_v_head, | |
| float* d_pool_q, | |
| float* d_pool_q_head, | |
| float* d_pool_scores, | |
| float* d_pool_ctx_head, | |
| float* d_pool_context | |
| ); | |
| void conv2d_3x3_forward( | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_in, | |
| const float* d_weight, | |
| const float* d_bias, | |
| float* d_out, | |
| float* d_im2col, | |
| int in_c, int in_h, int in_w, | |
| int out_c, int stride, int pad | |
| ); | |