#pragma once #include #include #include #define WARP_SIZE 32 // ------------------------------------------------------------- // 1. Dual-POV Frame Preprocessing Kernel // ------------------------------------------------------------- // Ingests HWC uint8 raw agentview & wristview frames (256x256x3) // Performs Robosuite [::-1, ::-1] camera flip and [-1, 1] normalization // Writes contiguous [6, 256, 256] float tensor directly to GPU memory __global__ void preprocess_frames_kernel( const uint8_t* __restrict__ agent_img, const uint8_t* __restrict__ wrist_img, float* __restrict__ out_obs, int H, int W ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total_pixels = H * W; if (idx >= total_pixels) return; int r = idx / W; int c = idx % W; // Robosuite [::-1, ::-1] flip int flipped_r = H - 1 - r; int flipped_c = W - 1 - c; int in_idx = (flipped_r * W + flipped_c) * 3; // Agent (Channels 0, 1, 2) out_obs[0 * total_pixels + idx] = ((float)agent_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f; out_obs[1 * total_pixels + idx] = ((float)agent_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f; out_obs[2 * total_pixels + idx] = ((float)agent_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f; // Wrist (Channels 3, 4, 5) out_obs[3 * total_pixels + idx] = ((float)wrist_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f; out_obs[4 * total_pixels + idx] = ((float)wrist_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f; out_obs[5 * total_pixels + idx] = ((float)wrist_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f; } // ------------------------------------------------------------- // 2. LayerNorm Kernel (dim=256) // ------------------------------------------------------------- __global__ void layernorm_256_kernel( const float* __restrict__ in, const float* __restrict__ gamma, const float* __restrict__ beta, float* __restrict__ out, int num_rows, float eps = 1e-5f ) { int row = blockIdx.x; if (row >= num_rows) return; int tid = threadIdx.x; const float* in_row = in + row * 256; float* out_row = out + row * 256; float val = (tid < 256) ? in_row[tid] : 0.0f; // Compute mean __shared__ float s_mean; __shared__ float s_var; // Warp-level reduction for 256 floats (8 warps of 32 threads) float sum = val; #pragma unroll for (int offset = 16; offset > 0; offset /= 2) { sum += __shfl_down_sync(0xffffffff, sum, offset); } __shared__ float warp_sums[8]; int lane = tid % 32; int warp_id = tid / 32; if (lane == 0) warp_sums[warp_id] = sum; __syncthreads(); if (tid == 0) { float total_sum = 0.0f; for (int i = 0; i < 8; ++i) total_sum += warp_sums[i]; s_mean = total_sum / 256.0f; } __syncthreads(); float mean = s_mean; float diff = val - mean; float sq_diff = diff * diff; #pragma unroll for (int offset = 16; offset > 0; offset /= 2) { sq_diff += __shfl_down_sync(0xffffffff, sq_diff, offset); } if (lane == 0) warp_sums[warp_id] = sq_diff; __syncthreads(); if (tid == 0) { float total_sq = 0.0f; for (int i = 0; i < 8; ++i) total_sq += warp_sums[i]; s_var = total_sq / 256.0f; } __syncthreads(); float inv_std = rsqrtf(s_var + eps); float g = (gamma != nullptr) ? gamma[tid] : 1.0f; float b = (beta != nullptr) ? beta[tid] : 0.0f; out_row[tid] = (diff * inv_std) * g + b; } // ------------------------------------------------------------- // 3. GELU Activation Kernel // ------------------------------------------------------------- __global__ void gelu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx >= N) return; float x = in[idx]; // Exact PyTorch GELU: x * 0.5 * (1 + erf(x / sqrt(2))) out[idx] = x * 0.5f * (1.0f + erff(x * 0.7071067811865475f)); } // ------------------------------------------------------------- // 4. SiLU Activation Kernel // ------------------------------------------------------------- __global__ void silu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx >= N) return; float x = in[idx]; out[idx] = x / (1.0f + expf(-x)); } // ------------------------------------------------------------- // 5. FiLM Modulation Kernel: out = norm_h * (1.0 + scale) + shift // ------------------------------------------------------------- __global__ void film_modulate_256_kernel( const float* __restrict__ h_norm, const float* __restrict__ film_cond, // [512] -> scale (0:256), shift (256:512) float* __restrict__ out, int num_tokens ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx >= num_tokens * 256) return; int d = idx % 256; float scale = film_cond[d]; float shift = film_cond[256 + d]; out[idx] = h_norm[idx] * (1.0f + scale) + shift; } // ------------------------------------------------------------- // 6. Softmax Kernel (num_rows, num_cols <= 32) // ------------------------------------------------------------- __global__ void softmax_small_kernel( float* __restrict__ data, int num_rows, int num_cols ) { int row = blockIdx.x; if (row >= num_rows) return; int tid = threadIdx.x; float* row_data = data + row * num_cols; float val = (tid < num_cols) ? row_data[tid] : -1e20f; // Max reduction float max_val = val; #pragma unroll for (int offset = 16; offset > 0; offset /= 2) { max_val = fmaxf(max_val, __shfl_down_sync(0xffffffff, max_val, offset)); } max_val = __shfl_sync(0xffffffff, max_val, 0); float exp_val = (tid < num_cols) ? expf(val - max_val) : 0.0f; // Sum reduction float sum = exp_val; #pragma unroll for (int offset = 16; offset > 0; offset /= 2) { sum += __shfl_down_sync(0xffffffff, sum, offset); } sum = __shfl_sync(0xffffffff, sum, 0); if (tid < num_cols) { row_data[tid] = exp_val / (sum + 1e-12f); } } // ------------------------------------------------------------- // 7. Euler Integration Step: z = z + dt * v // ------------------------------------------------------------- __global__ void euler_step_kernel( float* __restrict__ z, const float* __restrict__ v, float dt, int N ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx >= N) return; z[idx] += dt * v[idx]; } // ------------------------------------------------------------- // 8. Action Denormalization: a = ((z + 1)/2) * (max - min) + min // ------------------------------------------------------------- __global__ void denormalize_actions_kernel( const float* __restrict__ z_chunk, const float* __restrict__ act_min, const float* __restrict__ act_max, float* __restrict__ out_actions, int chunk_size, int act_dim ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx >= chunk_size * act_dim) return; int dim_idx = idx % act_dim; float z = z_chunk[idx]; float mn = act_min[dim_idx]; float mx = act_max[dim_idx]; // Standard denormalization out_actions[idx] = ((z + 1.0f) * 0.5f) * (mx - mn) + mn; } // ------------------------------------------------------------- // 9. Sinusoidal Timestep Embedding (dim=256) // ------------------------------------------------------------- __global__ void sinusoidal_embedding_kernel(float t, float* __restrict__ out_emb, int dim = 256) { int tid = threadIdx.x; if (tid >= dim / 2) return; int half = dim / 2; float freq = expf(-logf(10000.0f) * (float)tid / (float)half); float arg = t * freq * 1000.0f; out_emb[tid] = sinf(arg); out_emb[half + tid] = cosf(arg); } // ------------------------------------------------------------- // 10. Transpose Packed QKV to Head-Major // ------------------------------------------------------------- __global__ void transpose_qkv_to_heads_kernel( const float* __restrict__ qkv, float* __restrict__ q_head, float* __restrict__ k_head, float* __restrict__ v_head, int N, int H, int D ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int total = N * H * D; if (idx >= total) return; int d = idx % D; int h = (idx / D) % H; int n = idx / (H * D); int row_stride = 3 * H * D; int hd = H * D; int q_in = n * row_stride + h * D + d; int k_in = n * row_stride + hd + h * D + d; int v_in = n * row_stride + 2 * hd + h * D + d; int out_idx = h * (N * D) + n * D + d; q_head[out_idx] = qkv[q_in]; k_head[out_idx] = qkv[k_in]; v_head[out_idx] = qkv[v_in]; } // ------------------------------------------------------------- // 11. Clamp Kernel // ------------------------------------------------------------- __global__ void clamp_kernel(float* data, float min_val, float max_val, int N) { int idx = blockIdx.x * blockDim.x + threadIdx.x; if (idx < N) { data[idx] = fminf(max_val, fmaxf(min_val, data[idx])); } }