Download c_engine/vision_encoder.cu from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 22 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/vision_encoder.cu
- Command line
-
hf download hf://aasivas/Parvulus/c_engine/vision_encoder.cu
-
curl -L -o vision_encoder.cu https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/vision_encoder.cu
22 kB
| #include "vision_encoder.cuh" | |
| #include "kernels.cuh" | |
| #include "parvulus_engine.h" | |
| #include <iostream> | |
| // ----------------------------------------------------------------------------- | |
| // Bias & Transposition Kernels | |
| // ----------------------------------------------------------------------------- | |
| __global__ void add_bias_1d_kernel(float* data, const float* bias, int dim) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| if (idx < dim) { | |
| data[idx] += bias[idx]; | |
| } | |
| } | |
| __global__ void add_bias_across_tokens_kernel(float* data, const float* bias, int num_tokens, int dim) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| int total = num_tokens * dim; | |
| if (idx < total) { | |
| int d = idx % dim; | |
| data[idx] += bias[d]; | |
| } | |
| } | |
| __global__ void transpose_interleaved_to_head_kernel( | |
| const float* __restrict__ in, | |
| float* __restrict__ out, | |
| int N, int H, int D | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| int total = N * H * D; | |
| if (idx >= total) return; | |
| int d = idx % D; | |
| int h = (idx / D) % H; | |
| int n = idx / (H * D); | |
| int in_idx = idx; | |
| int out_idx = h * (N * D) + n * D + d; | |
| out[out_idx] = in[in_idx]; | |
| } | |
| __global__ void transpose_head_to_interleaved_kernel( | |
| const float* __restrict__ in, | |
| float* __restrict__ out, | |
| int N, int H, int D | |
| ) { | |
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |
| int total = N * H * D; | |
| if (idx >= total) return; | |
| int d = idx % D; | |
| int n = (idx / D) % N; | |
| int h = idx / (N * D); | |
| int out_idx = n * (H * D) + h * D + d; | |
| out[out_idx] = in[idx]; | |
| } | |
| // ----------------------------------------------------------------------------- | |
| // Convolution Primitives | |
| // ----------------------------------------------------------------------------- | |
| void conv2d_3x3_forward( | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_in, | |
| const float* d_weight, | |
| const float* d_bias, | |
| float* d_out, | |
| float* d_im2col, | |
| int in_c, int in_h, int in_w, | |
| int out_c, int stride, int pad | |
| ) { | |
| int out_h = (in_h + 2 * pad - 3) / stride + 1; | |
| int out_w = (in_w + 2 * pad - 3) / stride + 1; | |
| int total_spatial = out_h * out_w; | |
| int total_elements = in_c * total_spatial; | |
| int threads = 256; | |
| int blocks = (total_elements + threads - 1) / threads; | |
| im2col_3x3_kernel<<<blocks, threads, 0, stream>>>( | |
| d_in, d_im2col, in_c, in_h, in_w, pad, stride, out_h, out_w | |
| ); | |
| float alpha = 1.0f, beta = 0.0f; | |
| cublasSgemm( | |
| handle, | |
| CUBLAS_OP_N, CUBLAS_OP_N, | |
| total_spatial, out_c, in_c * 9, | |
| &alpha, | |
| d_im2col, total_spatial, | |
| d_weight, in_c * 9, | |
| &beta, | |
| d_out, total_spatial | |
| ); | |
| if (d_bias) { | |
| int total = out_c * total_spatial; | |
| add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>( | |
| d_out, d_bias, out_c, total_spatial | |
| ); | |
| } | |
| } | |
| void conv2d_1x1_forward( | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_in, | |
| const float* d_weight, | |
| const float* d_bias, | |
| float* d_out, | |
| int in_c, int spatial, int out_c | |
| ) { | |
| float alpha = 1.0f, beta = 0.0f; | |
| cublasSgemm( | |
| handle, | |
| CUBLAS_OP_N, CUBLAS_OP_N, | |
| spatial, out_c, in_c, | |
| &alpha, | |
| d_in, spatial, | |
| d_weight, in_c, | |
| &beta, | |
| d_out, spatial | |
| ); | |
| if (d_bias) { | |
| int total = out_c * spatial; | |
| add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>( | |
| d_out, d_bias, out_c, spatial | |
| ); | |
| } | |
| } | |
| // ----------------------------------------------------------------------------- | |
| // FiLM ResBlock 2D Forward | |
| // ----------------------------------------------------------------------------- | |
| void film_resblock_forward( | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_in, | |
| float* d_out, | |
| float* d_scratch1, | |
| float* d_scratch2, | |
| float* d_im2col, | |
| float* d_film_ss, | |
| const float* d_cond, | |
| const std::string& prefix, | |
| ParvulusEngine* engine, | |
| int in_c, int out_c, int H, int W | |
| ) { | |
| int spatial = H * W; | |
| // 1. Fast GroupNorm + SiLU on d_in -> d_scratch1 (64-block parallel reduction) | |
| float* g_n1 = engine->get_tensor(prefix + "norm1.weight"); | |
| float* b_n1 = engine->get_tensor(prefix + "norm1.bias"); | |
| cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream); | |
| fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_in, engine->get_gn_stats(), in_c, spatial); | |
| fast_groupnorm_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_in, engine->get_gn_stats(), g_n1, b_n1, d_scratch1, in_c, spatial, true); | |
| // 2. conv1 (3x3, in_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2 | |
| float* w_c1 = engine->get_tensor(prefix + "conv1.weight"); | |
| float* b_c1 = engine->get_tensor(prefix + "conv1.bias"); | |
| conv2d_3x3_forward(handle, stream, d_scratch1, w_c1, b_c1, d_scratch2, d_im2col, in_c, H, W, out_c, 1, 1); | |
| // 3. FiLM linear projection: Linear(256, 2 * out_c) | |
| // d_film_ss = cond * W^T + b | |
| float* w_film = engine->get_tensor(prefix + "film.weight"); | |
| float* b_film = engine->get_tensor(prefix + "film.bias"); | |
| float alpha = 1.0f, beta = 0.0f; | |
| cublasSgemv(handle, CUBLAS_OP_T, 256, 2 * out_c, &alpha, w_film, 256, d_cond, 1, &beta, d_film_ss, 1); | |
| add_bias_1d_kernel<<<(2 * out_c + 255) / 256, 256, 0, stream>>>(d_film_ss, b_film, 2 * out_c); | |
| // 4. Fast GroupNorm + FiLM modulation + SiLU on d_scratch2 -> d_scratch1 (64-block parallel reduction) | |
| float* g_n2 = engine->get_tensor(prefix + "norm2.weight"); | |
| float* b_n2 = engine->get_tensor(prefix + "norm2.bias"); | |
| float* d_scale = d_film_ss; | |
| float* d_shift = d_film_ss + out_c; | |
| cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream); | |
| fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch2, engine->get_gn_stats(), out_c, spatial); | |
| fast_groupnorm_film_silu_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>( | |
| d_scratch2, engine->get_gn_stats(), g_n2, b_n2, d_scale, d_shift, d_scratch1, out_c, spatial | |
| ); | |
| // 5. conv2 (3x3, out_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2 | |
| float* w_c2 = engine->get_tensor(prefix + "conv2.weight"); | |
| float* b_c2 = engine->get_tensor(prefix + "conv2.bias"); | |
| conv2d_3x3_forward(handle, stream, d_scratch1, w_c2, b_c2, d_scratch2, d_im2col, out_c, H, W, out_c, 1, 1); | |
| // 6. Skip connection | |
| if (in_c != out_c) { | |
| float* w_skip = engine->get_tensor(prefix + "skip.weight"); | |
| float* b_skip = engine->get_tensor(prefix + "skip.bias"); | |
| conv2d_1x1_forward(handle, stream, d_in, w_skip, b_skip, d_scratch1, in_c, spatial, out_c); | |
| add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>( | |
| d_scratch2, d_scratch1, d_out, out_c * spatial | |
| ); | |
| } else { | |
| add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>( | |
| d_scratch2, d_in, d_out, out_c * spatial | |
| ); | |
| } | |
| } | |
| // ------------------------------------------------------------- | |
| // Cross-View Mixer Forward | |
| // ------------------------------------------------------------- | |
| void cv_mixer_forward( | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_agent, | |
| const float* d_wrist, | |
| float* d_agent_out, | |
| float* d_wrist_out, | |
| float* d_mixed_out, | |
| float* d_scratch1, | |
| float* d_scratch2, | |
| float* d_im2col, | |
| const std::string& prefix, | |
| ParvulusEngine* engine, | |
| int ch, int H, int W | |
| ) { | |
| int spatial = H * W; | |
| // 1. Joint = cat(d_agent, d_wrist) -> shape [2*ch, spatial] in d_mixed_out | |
| cudaMemcpyAsync(d_mixed_out, d_agent, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); | |
| cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); | |
| // 2. context_conv: Conv1x1 (2*ch -> ch) on joint -> GroupNorm(8, ch) -> SiLU -> ctx | |
| float* w_ctx = engine->get_tensor(prefix + "context_conv.0.weight"); | |
| float* b_ctx = engine->get_tensor(prefix + "context_conv.0.bias"); | |
| conv2d_1x1_forward(handle, stream, d_mixed_out, w_ctx, b_ctx, d_scratch1, 2 * ch, spatial, ch); | |
| float* g_ctx = engine->get_tensor(prefix + "context_conv.1.weight"); | |
| float* b_ctx_gn = engine->get_tensor(prefix + "context_conv.1.bias"); | |
| float* d_ctx = d_scratch2; | |
| cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream); | |
| fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch1, engine->get_gn_stats(), ch, spatial); | |
| fast_groupnorm_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch1, engine->get_gn_stats(), g_ctx, b_ctx_gn, d_ctx, ch, spatial, true); | |
| // 3. agent_proj (3x3 on ctx) -> d_scratch1 | |
| float* w_ap = engine->get_tensor(prefix + "agent_proj.weight"); | |
| float* b_ap = engine->get_tensor(prefix + "agent_proj.bias"); | |
| conv2d_3x3_forward(handle, stream, d_ctx, w_ap, b_ap, d_scratch1, d_im2col, ch, H, W, ch, 1, 1); | |
| // 4. agent_gate (1x1 on ctx) -> d_agent_out (temporary gate buffer) | |
| float* w_ag = engine->get_tensor(prefix + "agent_gate.weight"); | |
| float* b_ag = engine->get_tensor(prefix + "agent_gate.bias"); | |
| conv2d_1x1_forward(handle, stream, d_ctx, w_ag, b_ag, d_agent_out, ch, spatial, ch); | |
| // f_agent_out = d_agent + d_scratch1 * sigmoid(d_agent_out) | |
| gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>( | |
| d_agent, d_scratch1, d_agent_out, d_agent_out, ch * spatial | |
| ); | |
| // 5. wrist_proj (3x3 on ctx) -> d_scratch1 | |
| float* w_wp = engine->get_tensor(prefix + "wrist_proj.weight"); | |
| float* b_wp = engine->get_tensor(prefix + "wrist_proj.bias"); | |
| conv2d_3x3_forward(handle, stream, d_ctx, w_wp, b_wp, d_scratch1, d_im2col, ch, H, W, ch, 1, 1); | |
| // 6. wrist_gate (1x1 on ctx) -> d_wrist_out (temporary gate buffer) | |
| float* w_wg = engine->get_tensor(prefix + "wrist_gate.weight"); | |
| float* b_wg = engine->get_tensor(prefix + "wrist_gate.bias"); | |
| conv2d_1x1_forward(handle, stream, d_ctx, w_wg, b_wg, d_wrist_out, ch, spatial, ch); | |
| // f_wrist_out = d_wrist + d_scratch1 * sigmoid(d_wrist_out) | |
| gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>( | |
| d_wrist, d_scratch1, d_wrist_out, d_wrist_out, ch * spatial | |
| ); | |
| // 7. f_mixed = cat(f_agent_out, f_wrist_out) | |
| cudaMemcpyAsync(d_mixed_out, d_agent_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); | |
| cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream); | |
| } | |
| // ------------------------------------------------------------- | |
| // Multi-Head Attention Pooling Forward (Fused Projections & Precomputed Q) | |
| // ------------------------------------------------------------- | |
| void attn_pool_forward( | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_mixed, | |
| float* d_out_tokens_4x256, | |
| float* d_k_proj, // [N, 256] | |
| float* d_v_proj, // [N, 256] | |
| float* d_k_head, // [4, N, 64] | |
| float* d_v_head, // [4, N, 64] | |
| float* d_scores, // [4, 4, N] | |
| float* d_ctx_head, // [4, 4, 64] | |
| float* d_context, // [4, 256] | |
| int stage, | |
| ParvulusEngine* engine, | |
| int in_c, int H, int W | |
| ) { | |
| int N = H * W; | |
| float alpha = 1.0f, beta = 0.0f; | |
| // 1. Fused K projection directly from d_mixed (eliminates in_proj GEMM & bias) | |
| float* w_k_fused = engine->get_attn_wk_fused(stage); | |
| float* b_k_fused = engine->get_attn_bk_fused(stage); | |
| cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_k_fused, in_c, d_mixed, N, &beta, d_k_proj, 256); | |
| add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_k_proj, b_k_fused, N, 256); | |
| transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>( | |
| d_k_proj, d_k_head, N, 4, 64 | |
| ); | |
| // 2. Fused V projection directly from d_mixed (eliminates in_proj GEMM & bias) | |
| float* w_v_fused = engine->get_attn_wv_fused(stage); | |
| float* b_v_fused = engine->get_attn_bv_fused(stage); | |
| cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_v_fused, in_c, d_mixed, N, &beta, d_v_proj, 256); | |
| add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_v_proj, b_v_fused, N, 256); | |
| transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>( | |
| d_v_proj, d_v_head, N, 4, 64 | |
| ); | |
| // 3. Precomputed Q head (Zero FLOPs, Zero runtime GEMM) | |
| float* d_q_head = engine->get_attn_q_head(stage); | |
| // 4. Batched GEMM: Scores = Q * K^T * 0.125 | |
| float scale = 0.125f; | |
| cublasSgemmStridedBatched( | |
| handle, | |
| CUBLAS_OP_T, CUBLAS_OP_N, | |
| N, 4, 64, | |
| &scale, | |
| d_k_head, 64, 64 * N, | |
| d_q_head, 64, 4 * 64, | |
| &beta, | |
| d_scores, N, 4 * N, | |
| 4 | |
| ); | |
| // 5. Softmax over N keys for each head & query | |
| attn_pool_softmax_kernel<<<dim3(4, 4), 256, 0, stream>>>(d_scores, N); | |
| // 6. Context_h = Scores_h * V_h | |
| // Scores_h: (N x 4 col-major) | |
| // V_h: (64 x N col-major, lda = 64) | |
| // Context_h = V_h * Scores_h: (64 x N) x (N x 4) = (64 x 4 col-major, ldc = 64) | |
| cublasSgemmStridedBatched( | |
| handle, | |
| CUBLAS_OP_N, CUBLAS_OP_N, | |
| 64, 4, N, | |
| &alpha, | |
| d_v_head, 64, 64 * N, | |
| d_scores, N, 4 * N, | |
| &beta, | |
| d_ctx_head, 64, 4 * 64, | |
| 4 | |
| ); | |
| // Transpose context from head-major [4 heads, 4 queries, 64 dims] to interleaved [4 queries, 256 dims] | |
| transpose_head_to_interleaved_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream>>>( | |
| d_ctx_head, d_context, 4, 4, 64 | |
| ); | |
| // 7. Out projection: | |
| std::string prefix = "encoder.attn_poolers." + std::to_string(stage) + "."; | |
| float* w_out = engine->get_tensor(prefix + "cross_attn.out_proj.weight"); | |
| float* b_out = engine->get_tensor(prefix + "cross_attn.out_proj.bias"); | |
| cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_out, 256, d_context, 256, &beta, d_out_tokens_4x256, 256); | |
| add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream>>>(d_out_tokens_4x256, b_out, 4, 256); | |
| // 8. LayerNorm(256) | |
| float* g_norm = engine->get_tensor(prefix + "norm.weight"); | |
| float* b_norm = engine->get_tensor(prefix + "norm.bias"); | |
| layernorm_256_kernel<<<4, 256, 0, stream>>>(d_out_tokens_4x256, g_norm, b_norm, d_out_tokens_4x256, 4); | |
| } | |
| // ----------------------------------------------------------------------------- | |
| // Complete DualView Vision Encoder Forward Pass | |
| // ----------------------------------------------------------------------------- | |
| void dual_view_encoder_forward( | |
| ParvulusEngine* engine, | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_obs, | |
| const float* d_cond, | |
| float* d_z_vis, | |
| float* d_scratch_a, | |
| float* d_scratch_b, | |
| float* d_scratch_c, | |
| float* d_scratch_d, | |
| float* d_im2col, | |
| float* d_film_ss, | |
| float* d_pool_k, | |
| float* d_pool_v, | |
| float* d_pool_k_head, | |
| float* d_pool_v_head, | |
| float* d_pool_q, | |
| float* d_pool_q_head, | |
| float* d_pool_scores, | |
| float* d_pool_ctx_head, | |
| float* d_pool_context | |
| ) { | |
| const float* d_agent_raw = d_obs; | |
| const float* d_wrist_raw = d_obs + 3 * 256 * 256; | |
| // 1. Stem convolutions: Conv3x3 (3 -> 48, pad=1, stride=1) on 256x256 | |
| float* w_a_stem = engine->get_tensor("encoder.agent_stem.weight"); | |
| float* b_a_stem = engine->get_tensor("encoder.agent_stem.bias"); | |
| conv2d_3x3_forward(handle, stream, d_agent_raw, w_a_stem, b_a_stem, d_scratch_a, d_im2col, 3, 256, 256, 48, 1, 1); | |
| float* w_w_stem = engine->get_tensor("encoder.wrist_stem.weight"); | |
| float* b_w_stem = engine->get_tensor("encoder.wrist_stem.bias"); | |
| conv2d_3x3_forward(handle, stream, d_wrist_raw, w_w_stem, b_w_stem, d_scratch_b, d_im2col, 3, 256, 256, 48, 1, 1); | |
| // 2. Downsample stem 0.5x (256x256 -> 128x128): | |
| avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_a, d_scratch_c, 48, 128, 128); | |
| avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_b, d_scratch_d, 48, 128, 128); | |
| int channel_mults[4] = {1, 2, 2, 4}; | |
| int in_channels[4] = {48, 48, 96, 96}; | |
| int resolutions[4] = {128, 64, 32, 16}; | |
| for (int stage = 0; stage < 4; ++stage) { | |
| int in_ch = in_channels[stage]; | |
| int out_ch = 48 * channel_mults[stage]; | |
| int H = resolutions[stage]; | |
| int W = resolutions[stage]; | |
| std::string pfx_a = "encoder.agent_stages." + std::to_string(stage) + ".0."; | |
| std::string pfx_w = "encoder.wrist_stages." + std::to_string(stage) + ".0."; | |
| std::string pfx_mix = "encoder.cv_mixers." + std::to_string(stage) + "."; | |
| std::string pfx_pool = "encoder.attn_poolers." + std::to_string(stage) + "."; | |
| std::string pfx_down = "encoder.downsamples." + std::to_string(stage) + "."; | |
| float* d_stage_tokens = d_z_vis + stage * 4 * 256; | |
| if (stage % 2 == 0) { | |
| // Inputs are in d_scratch_c (agent) and d_scratch_d (wrist) | |
| // (a) FiLM ResBlocks -> write to d_scratch_a and d_scratch_b | |
| film_resblock_forward( | |
| handle, stream, d_scratch_c, d_scratch_a, d_pool_k, d_pool_v, | |
| d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W | |
| ); | |
| film_resblock_forward( | |
| handle, stream, d_scratch_d, d_scratch_b, d_pool_k, d_pool_v, | |
| d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W | |
| ); | |
| // (b) CVMixer: inputs (a, b) -> agent_out (c), wrist_out (d), mixed_out (a) | |
| cv_mixer_forward( | |
| handle, stream, d_scratch_a, d_scratch_b, d_scratch_c, d_scratch_d, d_scratch_a, | |
| d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W | |
| ); | |
| // (c) AttnPool2d: mixed is in d_scratch_a | |
| attn_pool_forward( | |
| handle, stream, d_scratch_a, d_stage_tokens, | |
| d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head, | |
| d_pool_scores, d_pool_ctx_head, d_pool_context, | |
| stage, engine, 2 * out_ch, H, W | |
| ); | |
| // (d) Downsample: read (c, d) -> write to (a, b) | |
| if (stage < 3) { | |
| float* w_down = engine->get_tensor(pfx_down + "weight"); | |
| float* b_down = engine->get_tensor(pfx_down + "bias"); | |
| conv2d_3x3_forward(handle, stream, d_scratch_c, w_down, b_down, d_scratch_a, d_im2col, out_ch, H, W, out_ch, 2, 1); | |
| conv2d_3x3_forward(handle, stream, d_scratch_d, w_down, b_down, d_scratch_b, d_im2col, out_ch, H, W, out_ch, 2, 1); | |
| } | |
| } else { | |
| // Inputs are in d_scratch_a (agent) and d_scratch_b (wrist) | |
| // (a) FiLM ResBlocks -> write to d_scratch_c and d_scratch_d | |
| film_resblock_forward( | |
| handle, stream, d_scratch_a, d_scratch_c, d_pool_k, d_pool_v, | |
| d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W | |
| ); | |
| film_resblock_forward( | |
| handle, stream, d_scratch_b, d_scratch_d, d_pool_k, d_pool_v, | |
| d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W | |
| ); | |
| // (b) CVMixer: inputs (c, d) -> agent_out (a), wrist_out (b), mixed_out (c) | |
| cv_mixer_forward( | |
| handle, stream, d_scratch_c, d_scratch_d, d_scratch_a, d_scratch_b, d_scratch_c, | |
| d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W | |
| ); | |
| // (c) AttnPool2d: mixed is in d_scratch_c | |
| attn_pool_forward( | |
| handle, stream, d_scratch_c, d_stage_tokens, | |
| d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head, | |
| d_pool_scores, d_pool_ctx_head, d_pool_context, | |
| stage, engine, 2 * out_ch, H, W | |
| ); | |
| // (d) Downsample: read (a, b) -> write to (c, d) | |
| if (stage < 3) { | |
| float* w_down = engine->get_tensor(pfx_down + "weight"); | |
| float* b_down = engine->get_tensor(pfx_down + "bias"); | |
| conv2d_3x3_forward(handle, stream, d_scratch_a, w_down, b_down, d_scratch_c, d_im2col, out_ch, H, W, out_ch, 2, 1); | |
| conv2d_3x3_forward(handle, stream, d_scratch_b, w_down, b_down, d_scratch_d, d_im2col, out_ch, H, W, out_ch, 2, 1); | |
| } | |
| } | |
| } | |
| } | |
| void test_groupnorm( | |
| const float* d_in, const float* gamma, const float* beta, float* d_out, | |
| int C, int H, int W, bool silu, cudaStream_t stream | |
| ) { | |
| groupnorm_2d_kernel<<<8, 256, 0, stream>>>(d_in, gamma, beta, d_out, C, H, W, silu); | |
| } | |
| void test_film_resblock( | |
| ParvulusEngine* engine, | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_in, | |
| float* d_out, | |
| float* d_scratch1, | |
| float* d_scratch2, | |
| float* d_im2col, | |
| float* d_film_ss, | |
| const float* d_cond, | |
| const std::string& prefix, | |
| int in_c, int out_c, int H, int W | |
| ) { | |
| film_resblock_forward( | |
| handle, stream, d_in, d_out, d_scratch1, d_scratch2, d_im2col, d_film_ss, | |
| d_cond, prefix, engine, in_c, out_c, H, W | |
| ); | |
| } | |
| void test_attn_pool( | |
| ParvulusEngine* engine, | |
| cublasHandle_t handle, | |
| cudaStream_t stream, | |
| const float* d_mixed, | |
| float* d_out_tokens, | |
| float* d_scratch_b, | |
| float* d_pool_k, | |
| float* d_pool_v, | |
| float* d_pool_k_head, | |
| float* d_pool_v_head, | |
| float* d_pool_q, | |
| float* d_pool_q_head, | |
| float* d_pool_scores, | |
| float* d_pool_ctx_head, | |
| float* d_pool_context, | |
| const std::string& prefix, | |
| int in_c, int H, int W | |
| ) { | |
| attn_pool_forward( | |
| handle, stream, d_mixed, d_out_tokens, | |
| d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head, | |
| d_pool_scores, d_pool_ctx_head, d_pool_context, | |
| 0, engine, in_c, H, W | |
| ); | |
| } | |