Parvulus / c_engine /vision_encoder.cuh
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
16.3 kB
#pragma once
#include <cuda_runtime.h>
#include <cublas_v2.h>
#include <cmath>
// -----------------------------------------------------------------------------
// CUDA Kernels for Vision Encoder
// -----------------------------------------------------------------------------
__global__ void im2col_3x3_kernel(
const float* __restrict__ data_im,
float* __restrict__ data_col,
int channels, int height, int width,
int pad, int stride,
int out_h, int out_w
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total_spatial = out_h * out_w;
int total = channels * total_spatial;
if (idx >= total) return;
int spatial_idx = idx % total_spatial;
int c = idx / total_spatial;
int w_out = spatial_idx % out_w;
int h_out = spatial_idx / out_w;
int h_in_base = h_out * stride - pad;
int w_in_base = w_out * stride - pad;
const float* im_c = data_im + c * (height * width);
int col_channel_base = c * 9;
#pragma unroll
for (int kh = 0; kh < 3; ++kh) {
int h_in = h_in_base + kh;
#pragma unroll
for (int kw = 0; kw < 3; ++kw) {
int w_in = w_in_base + kw;
float val = 0.0f;
if (h_in >= 0 && h_in < height && w_in >= 0 && w_in < width) {
val = im_c[h_in * width + w_in];
}
int col_idx = (col_channel_base + kh * 3 + kw) * total_spatial + spatial_idx;
data_col[col_idx] = val;
}
}
}
__global__ void add_bias_2d_kernel(
float* __restrict__ data,
const float* __restrict__ bias,
int channels, int spatial_dim
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total = channels * spatial_dim;
if (idx >= total) return;
int c = idx / spatial_dim;
data[idx] += bias[c];
}
__global__ void silu_kernel(float* data, int total) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < total) {
float x = data[idx];
data[idx] = x / (1.0f + __expf(-x));
}
}
__global__ void avg_pool_2x2_kernel(
const float* __restrict__ in,
float* __restrict__ out,
int C, int H_out, int W_out
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total = C * H_out * W_out;
if (idx >= total) return;
int w = idx % W_out;
int h = (idx / W_out) % H_out;
int c = idx / (H_out * W_out);
int in_w = w * 2;
int in_h = h * 2;
int in_w_total = W_out * 2;
int in_h_total = H_out * 2;
const float* in_c = in + c * (in_h_total * in_w_total);
float val = 0.25f * (
in_c[in_h * in_w_total + in_w] +
in_c[in_h * in_w_total + in_w + 1] +
in_c[(in_h + 1) * in_w_total + in_w] +
in_c[(in_h + 1) * in_w_total + in_w + 1]
);
out[idx] = val;
}
__global__ void gated_residual_kernel(
const float* __restrict__ f_base,
const float* __restrict__ f_proj,
const float* __restrict__ f_gate,
float* __restrict__ f_out,
int total
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < total) {
float gate = 1.0f / (1.0f + __expf(-f_gate[idx]));
f_out[idx] = f_base[idx] + f_proj[idx] * gate;
}
}
__global__ void add_residual_kernel(
const float* __restrict__ a,
const float* __restrict__ b,
float* __restrict__ out,
int total
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < total) {
out[idx] = a[idx] + b[idx];
}
}
// GroupNorm(num_groups=8) + optional SiLU
__global__ void groupnorm_2d_kernel(
const float* __restrict__ in,
const float* __restrict__ gamma,
const float* __restrict__ beta,
float* __restrict__ out,
int C, int H, int W,
bool apply_silu
) {
// 1 block per group (8 groups total)
int group_idx = blockIdx.x;
int num_groups = 8;
int channels_per_group = C / num_groups;
int spatial = H * W;
int M = channels_per_group * spatial;
int c_start = group_idx * channels_per_group;
int tid = threadIdx.x;
int bdim = blockDim.x;
// Pass 1: compute mean and variance
float local_sum = 0.0f;
float local_sq_sum = 0.0f;
for (int i = tid; i < M; i += bdim) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
float x = in[c * spatial + s];
local_sum += x;
local_sq_sum += x * x;
}
// Warp shuffle reduction
for (int offset = 16; offset > 0; offset /= 2) {
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset);
}
__shared__ float s_sum[32];
__shared__ float s_sq_sum[32];
int lane = tid % 32;
int wid = tid / 32;
if (lane == 0) {
s_sum[wid] = local_sum;
s_sq_sum[wid] = local_sq_sum;
}
__syncthreads();
if (wid == 0) {
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f;
for (int offset = 16; offset > 0; offset /= 2) {
val += __shfl_down_sync(0xffffffff, val, offset);
val_sq += __shfl_down_sync(0xffffffff, val_sq, offset);
}
if (lane == 0) {
s_sum[0] = val;
s_sq_sum[0] = val_sq;
}
}
__syncthreads();
float mean = s_sum[0] / (float)M;
float var = (s_sq_sum[0] / (float)M) - (mean * mean);
if (var < 0.0f) var = 0.0f;
float inv_std = rsqrtf(var + 1e-5f);
// Pass 2: Normalize and write output
for (int i = tid; i < M; i += bdim) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
int idx = c * spatial + s;
float x = in[idx];
float norm_x = (x - mean) * inv_std;
float y = norm_x * gamma[c] + beta[c];
if (apply_silu) {
y = y / (1.0f + __expf(-y));
}
out[idx] = y;
}
}
// GroupNorm + FiLM scale/shift + SiLU in a single fused kernel
__global__ void groupnorm_film_silu_2d_kernel(
const float* __restrict__ in,
const float* __restrict__ gamma,
const float* __restrict__ beta,
const float* __restrict__ scale,
const float* __restrict__ shift,
float* __restrict__ out,
int C, int H, int W
) {
int group_idx = blockIdx.x;
int num_groups = 8;
int channels_per_group = C / num_groups;
int spatial = H * W;
int M = channels_per_group * spatial;
int c_start = group_idx * channels_per_group;
int tid = threadIdx.x;
int bdim = blockDim.x;
float local_sum = 0.0f;
float local_sq_sum = 0.0f;
for (int i = tid; i < M; i += bdim) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
float x = in[c * spatial + s];
local_sum += x;
local_sq_sum += x * x;
}
for (int offset = 16; offset > 0; offset /= 2) {
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset);
}
__shared__ float s_sum[32];
__shared__ float s_sq_sum[32];
int lane = tid % 32;
int wid = tid / 32;
if (lane == 0) {
s_sum[wid] = local_sum;
s_sq_sum[wid] = local_sq_sum;
}
__syncthreads();
if (wid == 0) {
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f;
for (int offset = 16; offset > 0; offset /= 2) {
val += __shfl_down_sync(0xffffffff, val, offset);
val_sq += __shfl_down_sync(0xffffffff, val_sq, offset);
}
if (lane == 0) {
s_sum[0] = val;
s_sq_sum[0] = val_sq;
}
}
__syncthreads();
float mean = s_sum[0] / (float)M;
float var = (s_sq_sum[0] / (float)M) - (mean * mean);
if (var < 0.0f) var = 0.0f;
float inv_std = rsqrtf(var + 1e-5f);
for (int i = tid; i < M; i += bdim) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
int idx = c * spatial + s;
float x = in[idx];
float norm_x = (x - mean) * inv_std;
// GroupNorm affine
float gn_out = norm_x * gamma[c] + beta[c];
// FiLM modulation: gn_out * (1 + scale) + shift
float film_out = gn_out * (1.0f + scale[c]) + shift[c];
// SiLU
out[idx] = film_out / (1.0f + __expf(-film_out));
}
}
// -------------------------------------------------------------
// Parallel High-Occupancy GroupNorm (64 blocks: 8 groups x 8 workers)
// -------------------------------------------------------------
__global__ void fast_groupnorm_stat_kernel(
const float* __restrict__ in,
float* __restrict__ group_stats,
int C, int spatial
) {
int group_idx = blockIdx.x;
int block_in_group = blockIdx.y;
int num_blocks_per_group = gridDim.y;
int channels_per_group = C / 8;
int M = channels_per_group * spatial;
int c_start = group_idx * channels_per_group;
int tid = threadIdx.x;
int bdim = blockDim.x;
float local_sum = 0.0f;
float local_sq_sum = 0.0f;
int stride = bdim * num_blocks_per_group;
int start = block_in_group * bdim + tid;
for (int i = start; i < M; i += stride) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
float x = in[c * spatial + s];
local_sum += x;
local_sq_sum += x * x;
}
for (int offset = 16; offset > 0; offset /= 2) {
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset);
}
__shared__ float s_sum[32];
__shared__ float s_sq_sum[32];
int lane = tid % 32;
int wid = tid / 32;
if (lane == 0) {
s_sum[wid] = local_sum;
s_sq_sum[wid] = local_sq_sum;
}
__syncthreads();
if (wid == 0) {
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f;
for (int offset = 16; offset > 0; offset /= 2) {
val += __shfl_down_sync(0xffffffff, val, offset);
val_sq += __shfl_down_sync(0xffffffff, val_sq, offset);
}
if (lane == 0) {
atomicAdd(&group_stats[group_idx * 2 + 0], val);
atomicAdd(&group_stats[group_idx * 2 + 1], val_sq);
}
}
}
__global__ void fast_groupnorm_apply_kernel(
const float* __restrict__ in,
const float* __restrict__ group_stats,
const float* __restrict__ gamma,
const float* __restrict__ beta,
float* __restrict__ out,
int C, int spatial,
bool apply_silu
) {
int group_idx = blockIdx.x;
int block_in_group = blockIdx.y;
int num_blocks_per_group = gridDim.y;
int channels_per_group = C / 8;
int M = channels_per_group * spatial;
int c_start = group_idx * channels_per_group;
float sum = group_stats[group_idx * 2 + 0];
float sq_sum = group_stats[group_idx * 2 + 1];
float mean = sum / (float)M;
float var = (sq_sum / (float)M) - (mean * mean);
float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f);
int tid = threadIdx.x;
int bdim = blockDim.x;
int stride = bdim * num_blocks_per_group;
int start = block_in_group * bdim + tid;
for (int i = start; i < M; i += stride) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
int idx = c * spatial + s;
float x = in[idx];
float norm = (x - mean) * inv_std;
float y = norm * gamma[c] + beta[c];
if (apply_silu) {
y = y * (1.0f / (1.0f + __expf(-y)));
}
out[idx] = y;
}
}
__global__ void fast_groupnorm_film_silu_apply_kernel(
const float* __restrict__ in,
const float* __restrict__ group_stats,
const float* __restrict__ gamma,
const float* __restrict__ beta,
const float* __restrict__ scale,
const float* __restrict__ shift,
float* __restrict__ out,
int C, int spatial
) {
int group_idx = blockIdx.x;
int block_in_group = blockIdx.y;
int num_blocks_per_group = gridDim.y;
int channels_per_group = C / 8;
int M = channels_per_group * spatial;
int c_start = group_idx * channels_per_group;
float sum = group_stats[group_idx * 2 + 0];
float sq_sum = group_stats[group_idx * 2 + 1];
float mean = sum / (float)M;
float var = (sq_sum / (float)M) - (mean * mean);
float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f);
int tid = threadIdx.x;
int bdim = blockDim.x;
int stride = bdim * num_blocks_per_group;
int start = block_in_group * bdim + tid;
for (int i = start; i < M; i += stride) {
int c_local = i / spatial;
int s = i % spatial;
int c = c_start + c_local;
int idx = c * spatial + s;
float x = in[idx];
float norm_x = (x - mean) * inv_std;
float h = norm_x * gamma[c] + beta[c];
float modulated = h * (1.0f + scale[c]) + shift[c];
float out_val = modulated * (1.0f / (1.0f + __expf(-modulated)));
out[idx] = out_val;
}
}
// Multi-head attention softmax kernel for AttnPool: 4 heads, 4 queries, N keys
__global__ void attn_pool_softmax_kernel(
float* __restrict__ scores,
int N // number of spatial tokens (e.g. 16384, 4096, 1024, 256)
) {
// blockIdx.x: query_idx in [0..3]
// blockIdx.y: head_idx in [0..3]
int q_idx = blockIdx.x;
int h_idx = blockIdx.y;
int tid = threadIdx.x;
int bdim = blockDim.x;
float* row = scores + (h_idx * 4 + q_idx) * N;
// Find max for numerical stability
float local_max = -1e20f;
for (int i = tid; i < N; i += bdim) {
local_max = fmaxf(local_max, row[i]);
}
for (int offset = 16; offset > 0; offset /= 2) {
local_max = fmaxf(local_max, __shfl_down_sync(0xffffffff, local_max, offset));
}
__shared__ float s_max[32];
int lane = tid % 32;
int wid = tid / 32;
if (lane == 0) s_max[wid] = local_max;
__syncthreads();
if (wid == 0) {
float val = (tid < (bdim / 32)) ? s_max[lane] : -1e20f;
for (int offset = 16; offset > 0; offset /= 2) {
val = fmaxf(val, __shfl_down_sync(0xffffffff, val, offset));
}
if (lane == 0) s_max[0] = val;
}
__syncthreads();
float row_max = s_max[0];
// Compute exp and sum
float local_sum = 0.0f;
for (int i = tid; i < N; i += bdim) {
float e = __expf(row[i] - row_max);
row[i] = e;
local_sum += e;
}
for (int offset = 16; offset > 0; offset /= 2) {
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
}
__shared__ float s_sum[32];
if (lane == 0) s_sum[wid] = local_sum;
__syncthreads();
if (wid == 0) {
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
for (int offset = 16; offset > 0; offset /= 2) {
val += __shfl_down_sync(0xffffffff, val, offset);
}
if (lane == 0) s_sum[0] = val;
}
__syncthreads();
float inv_sum = 1.0f / (s_sum[0] + 1e-12f);
// Normalize
for (int i = tid; i < N; i += bdim) {
row[i] *= inv_sum;
}
}
// -----------------------------------------------------------------------------
// Vision Encoder Forward Function Declaration
// -----------------------------------------------------------------------------
class ParvulusEngine;
void dual_view_encoder_forward(
ParvulusEngine* engine,
cublasHandle_t handle,
cudaStream_t stream,
const float* d_obs,
const float* d_cond,
float* d_z_vis,
float* d_scratch_a,
float* d_scratch_b,
float* d_scratch_c,
float* d_scratch_d,
float* d_im2col,
float* d_film_ss,
float* d_pool_k,
float* d_pool_v,
float* d_pool_k_head,
float* d_pool_v_head,
float* d_pool_q,
float* d_pool_q_head,
float* d_pool_scores,
float* d_pool_ctx_head,
float* d_pool_context
);
void conv2d_3x3_forward(
cublasHandle_t handle,
cudaStream_t stream,
const float* d_in,
const float* d_weight,
const float* d_bias,
float* d_out,
float* d_im2col,
int in_c, int in_h, int in_w,
int out_c, int stride, int pad
);