Parvulus / c_engine /kernels.cuh
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
9.14 kB
#pragma once
#include <cuda_runtime.h>
#include <cmath>
#include <cstdint>
#define WARP_SIZE 32
// -------------------------------------------------------------
// 1. Dual-POV Frame Preprocessing Kernel
// -------------------------------------------------------------
// Ingests HWC uint8 raw agentview & wristview frames (256x256x3)
// Performs Robosuite [::-1, ::-1] camera flip and [-1, 1] normalization
// Writes contiguous [6, 256, 256] float tensor directly to GPU memory
__global__ void preprocess_frames_kernel(
const uint8_t* __restrict__ agent_img,
const uint8_t* __restrict__ wrist_img,
float* __restrict__ out_obs,
int H, int W
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total_pixels = H * W;
if (idx >= total_pixels) return;
int r = idx / W;
int c = idx % W;
// Robosuite [::-1, ::-1] flip
int flipped_r = H - 1 - r;
int flipped_c = W - 1 - c;
int in_idx = (flipped_r * W + flipped_c) * 3;
// Agent (Channels 0, 1, 2)
out_obs[0 * total_pixels + idx] = ((float)agent_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f;
out_obs[1 * total_pixels + idx] = ((float)agent_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f;
out_obs[2 * total_pixels + idx] = ((float)agent_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f;
// Wrist (Channels 3, 4, 5)
out_obs[3 * total_pixels + idx] = ((float)wrist_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f;
out_obs[4 * total_pixels + idx] = ((float)wrist_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f;
out_obs[5 * total_pixels + idx] = ((float)wrist_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f;
}
// -------------------------------------------------------------
// 2. LayerNorm Kernel (dim=256)
// -------------------------------------------------------------
__global__ void layernorm_256_kernel(
const float* __restrict__ in,
const float* __restrict__ gamma,
const float* __restrict__ beta,
float* __restrict__ out,
int num_rows,
float eps = 1e-5f
) {
int row = blockIdx.x;
if (row >= num_rows) return;
int tid = threadIdx.x;
const float* in_row = in + row * 256;
float* out_row = out + row * 256;
float val = (tid < 256) ? in_row[tid] : 0.0f;
// Compute mean
__shared__ float s_mean;
__shared__ float s_var;
// Warp-level reduction for 256 floats (8 warps of 32 threads)
float sum = val;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
sum += __shfl_down_sync(0xffffffff, sum, offset);
}
__shared__ float warp_sums[8];
int lane = tid % 32;
int warp_id = tid / 32;
if (lane == 0) warp_sums[warp_id] = sum;
__syncthreads();
if (tid == 0) {
float total_sum = 0.0f;
for (int i = 0; i < 8; ++i) total_sum += warp_sums[i];
s_mean = total_sum / 256.0f;
}
__syncthreads();
float mean = s_mean;
float diff = val - mean;
float sq_diff = diff * diff;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
sq_diff += __shfl_down_sync(0xffffffff, sq_diff, offset);
}
if (lane == 0) warp_sums[warp_id] = sq_diff;
__syncthreads();
if (tid == 0) {
float total_sq = 0.0f;
for (int i = 0; i < 8; ++i) total_sq += warp_sums[i];
s_var = total_sq / 256.0f;
}
__syncthreads();
float inv_std = rsqrtf(s_var + eps);
float g = (gamma != nullptr) ? gamma[tid] : 1.0f;
float b = (beta != nullptr) ? beta[tid] : 0.0f;
out_row[tid] = (diff * inv_std) * g + b;
}
// -------------------------------------------------------------
// 3. GELU Activation Kernel
// -------------------------------------------------------------
__global__ void gelu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N) return;
float x = in[idx];
// Exact PyTorch GELU: x * 0.5 * (1 + erf(x / sqrt(2)))
out[idx] = x * 0.5f * (1.0f + erff(x * 0.7071067811865475f));
}
// -------------------------------------------------------------
// 4. SiLU Activation Kernel
// -------------------------------------------------------------
__global__ void silu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N) return;
float x = in[idx];
out[idx] = x / (1.0f + expf(-x));
}
// -------------------------------------------------------------
// 5. FiLM Modulation Kernel: out = norm_h * (1.0 + scale) + shift
// -------------------------------------------------------------
__global__ void film_modulate_256_kernel(
const float* __restrict__ h_norm,
const float* __restrict__ film_cond, // [512] -> scale (0:256), shift (256:512)
float* __restrict__ out,
int num_tokens
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= num_tokens * 256) return;
int d = idx % 256;
float scale = film_cond[d];
float shift = film_cond[256 + d];
out[idx] = h_norm[idx] * (1.0f + scale) + shift;
}
// -------------------------------------------------------------
// 6. Softmax Kernel (num_rows, num_cols <= 32)
// -------------------------------------------------------------
__global__ void softmax_small_kernel(
float* __restrict__ data,
int num_rows,
int num_cols
) {
int row = blockIdx.x;
if (row >= num_rows) return;
int tid = threadIdx.x;
float* row_data = data + row * num_cols;
float val = (tid < num_cols) ? row_data[tid] : -1e20f;
// Max reduction
float max_val = val;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
max_val = fmaxf(max_val, __shfl_down_sync(0xffffffff, max_val, offset));
}
max_val = __shfl_sync(0xffffffff, max_val, 0);
float exp_val = (tid < num_cols) ? expf(val - max_val) : 0.0f;
// Sum reduction
float sum = exp_val;
#pragma unroll
for (int offset = 16; offset > 0; offset /= 2) {
sum += __shfl_down_sync(0xffffffff, sum, offset);
}
sum = __shfl_sync(0xffffffff, sum, 0);
if (tid < num_cols) {
row_data[tid] = exp_val / (sum + 1e-12f);
}
}
// -------------------------------------------------------------
// 7. Euler Integration Step: z = z + dt * v
// -------------------------------------------------------------
__global__ void euler_step_kernel(
float* __restrict__ z,
const float* __restrict__ v,
float dt,
int N
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= N) return;
z[idx] += dt * v[idx];
}
// -------------------------------------------------------------
// 8. Action Denormalization: a = ((z + 1)/2) * (max - min) + min
// -------------------------------------------------------------
__global__ void denormalize_actions_kernel(
const float* __restrict__ z_chunk,
const float* __restrict__ act_min,
const float* __restrict__ act_max,
float* __restrict__ out_actions,
int chunk_size,
int act_dim
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx >= chunk_size * act_dim) return;
int dim_idx = idx % act_dim;
float z = z_chunk[idx];
float mn = act_min[dim_idx];
float mx = act_max[dim_idx];
// Standard denormalization
out_actions[idx] = ((z + 1.0f) * 0.5f) * (mx - mn) + mn;
}
// -------------------------------------------------------------
// 9. Sinusoidal Timestep Embedding (dim=256)
// -------------------------------------------------------------
__global__ void sinusoidal_embedding_kernel(float t, float* __restrict__ out_emb, int dim = 256) {
int tid = threadIdx.x;
if (tid >= dim / 2) return;
int half = dim / 2;
float freq = expf(-logf(10000.0f) * (float)tid / (float)half);
float arg = t * freq * 1000.0f;
out_emb[tid] = sinf(arg);
out_emb[half + tid] = cosf(arg);
}
// -------------------------------------------------------------
// 10. Transpose Packed QKV to Head-Major
// -------------------------------------------------------------
__global__ void transpose_qkv_to_heads_kernel(
const float* __restrict__ qkv,
float* __restrict__ q_head,
float* __restrict__ k_head,
float* __restrict__ v_head,
int N, int H, int D
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total = N * H * D;
if (idx >= total) return;
int d = idx % D;
int h = (idx / D) % H;
int n = idx / (H * D);
int row_stride = 3 * H * D;
int hd = H * D;
int q_in = n * row_stride + h * D + d;
int k_in = n * row_stride + hd + h * D + d;
int v_in = n * row_stride + 2 * hd + h * D + d;
int out_idx = h * (N * D) + n * D + d;
q_head[out_idx] = qkv[q_in];
k_head[out_idx] = qkv[k_in];
v_head[out_idx] = qkv[v_in];
}
// -------------------------------------------------------------
// 11. Clamp Kernel
// -------------------------------------------------------------
__global__ void clamp_kernel(float* data, float min_val, float max_val, int N) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < N) {
data[idx] = fminf(max_val, fmaxf(min_val, data[idx]));
}
}