Parvulus / c_engine /vision_encoder.cu
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
22 kB
#include "vision_encoder.cuh"
#include "kernels.cuh"
#include "parvulus_engine.h"
#include <iostream>
// -----------------------------------------------------------------------------
// Bias & Transposition Kernels
// -----------------------------------------------------------------------------
__global__ void add_bias_1d_kernel(float* data, const float* bias, int dim) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
if (idx < dim) {
data[idx] += bias[idx];
}
}
__global__ void add_bias_across_tokens_kernel(float* data, const float* bias, int num_tokens, int dim) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total = num_tokens * dim;
if (idx < total) {
int d = idx % dim;
data[idx] += bias[d];
}
}
__global__ void transpose_interleaved_to_head_kernel(
const float* __restrict__ in,
float* __restrict__ out,
int N, int H, int D
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total = N * H * D;
if (idx >= total) return;
int d = idx % D;
int h = (idx / D) % H;
int n = idx / (H * D);
int in_idx = idx;
int out_idx = h * (N * D) + n * D + d;
out[out_idx] = in[in_idx];
}
__global__ void transpose_head_to_interleaved_kernel(
const float* __restrict__ in,
float* __restrict__ out,
int N, int H, int D
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int total = N * H * D;
if (idx >= total) return;
int d = idx % D;
int n = (idx / D) % N;
int h = idx / (N * D);
int out_idx = n * (H * D) + h * D + d;
out[out_idx] = in[idx];
}
// -----------------------------------------------------------------------------
// Convolution Primitives
// -----------------------------------------------------------------------------
void conv2d_3x3_forward(
cublasHandle_t handle,
cudaStream_t stream,
const float* d_in,
const float* d_weight,
const float* d_bias,
float* d_out,
float* d_im2col,
int in_c, int in_h, int in_w,
int out_c, int stride, int pad
) {
int out_h = (in_h + 2 * pad - 3) / stride + 1;
int out_w = (in_w + 2 * pad - 3) / stride + 1;
int total_spatial = out_h * out_w;
int total_elements = in_c * total_spatial;
int threads = 256;
int blocks = (total_elements + threads - 1) / threads;
im2col_3x3_kernel<<<blocks, threads, 0, stream>>>(
d_in, d_im2col, in_c, in_h, in_w, pad, stride, out_h, out_w
);
float alpha = 1.0f, beta = 0.0f;
cublasSgemm(
handle,
CUBLAS_OP_N, CUBLAS_OP_N,
total_spatial, out_c, in_c * 9,
&alpha,
d_im2col, total_spatial,
d_weight, in_c * 9,
&beta,
d_out, total_spatial
);
if (d_bias) {
int total = out_c * total_spatial;
add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>(
d_out, d_bias, out_c, total_spatial
);
}
}
void conv2d_1x1_forward(
cublasHandle_t handle,
cudaStream_t stream,
const float* d_in,
const float* d_weight,
const float* d_bias,
float* d_out,
int in_c, int spatial, int out_c
) {
float alpha = 1.0f, beta = 0.0f;
cublasSgemm(
handle,
CUBLAS_OP_N, CUBLAS_OP_N,
spatial, out_c, in_c,
&alpha,
d_in, spatial,
d_weight, in_c,
&beta,
d_out, spatial
);
if (d_bias) {
int total = out_c * spatial;
add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>(
d_out, d_bias, out_c, spatial
);
}
}
// -----------------------------------------------------------------------------
// FiLM ResBlock 2D Forward
// -----------------------------------------------------------------------------
void film_resblock_forward(
cublasHandle_t handle,
cudaStream_t stream,
const float* d_in,
float* d_out,
float* d_scratch1,
float* d_scratch2,
float* d_im2col,
float* d_film_ss,
const float* d_cond,
const std::string& prefix,
ParvulusEngine* engine,
int in_c, int out_c, int H, int W
) {
int spatial = H * W;
// 1. Fast GroupNorm + SiLU on d_in -> d_scratch1 (64-block parallel reduction)
float* g_n1 = engine->get_tensor(prefix + "norm1.weight");
float* b_n1 = engine->get_tensor(prefix + "norm1.bias");
cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream);
fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_in, engine->get_gn_stats(), in_c, spatial);
fast_groupnorm_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_in, engine->get_gn_stats(), g_n1, b_n1, d_scratch1, in_c, spatial, true);
// 2. conv1 (3x3, in_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2
float* w_c1 = engine->get_tensor(prefix + "conv1.weight");
float* b_c1 = engine->get_tensor(prefix + "conv1.bias");
conv2d_3x3_forward(handle, stream, d_scratch1, w_c1, b_c1, d_scratch2, d_im2col, in_c, H, W, out_c, 1, 1);
// 3. FiLM linear projection: Linear(256, 2 * out_c)
// d_film_ss = cond * W^T + b
float* w_film = engine->get_tensor(prefix + "film.weight");
float* b_film = engine->get_tensor(prefix + "film.bias");
float alpha = 1.0f, beta = 0.0f;
cublasSgemv(handle, CUBLAS_OP_T, 256, 2 * out_c, &alpha, w_film, 256, d_cond, 1, &beta, d_film_ss, 1);
add_bias_1d_kernel<<<(2 * out_c + 255) / 256, 256, 0, stream>>>(d_film_ss, b_film, 2 * out_c);
// 4. Fast GroupNorm + FiLM modulation + SiLU on d_scratch2 -> d_scratch1 (64-block parallel reduction)
float* g_n2 = engine->get_tensor(prefix + "norm2.weight");
float* b_n2 = engine->get_tensor(prefix + "norm2.bias");
float* d_scale = d_film_ss;
float* d_shift = d_film_ss + out_c;
cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream);
fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch2, engine->get_gn_stats(), out_c, spatial);
fast_groupnorm_film_silu_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(
d_scratch2, engine->get_gn_stats(), g_n2, b_n2, d_scale, d_shift, d_scratch1, out_c, spatial
);
// 5. conv2 (3x3, out_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2
float* w_c2 = engine->get_tensor(prefix + "conv2.weight");
float* b_c2 = engine->get_tensor(prefix + "conv2.bias");
conv2d_3x3_forward(handle, stream, d_scratch1, w_c2, b_c2, d_scratch2, d_im2col, out_c, H, W, out_c, 1, 1);
// 6. Skip connection
if (in_c != out_c) {
float* w_skip = engine->get_tensor(prefix + "skip.weight");
float* b_skip = engine->get_tensor(prefix + "skip.bias");
conv2d_1x1_forward(handle, stream, d_in, w_skip, b_skip, d_scratch1, in_c, spatial, out_c);
add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>(
d_scratch2, d_scratch1, d_out, out_c * spatial
);
} else {
add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>(
d_scratch2, d_in, d_out, out_c * spatial
);
}
}
// -------------------------------------------------------------
// Cross-View Mixer Forward
// -------------------------------------------------------------
void cv_mixer_forward(
cublasHandle_t handle,
cudaStream_t stream,
const float* d_agent,
const float* d_wrist,
float* d_agent_out,
float* d_wrist_out,
float* d_mixed_out,
float* d_scratch1,
float* d_scratch2,
float* d_im2col,
const std::string& prefix,
ParvulusEngine* engine,
int ch, int H, int W
) {
int spatial = H * W;
// 1. Joint = cat(d_agent, d_wrist) -> shape [2*ch, spatial] in d_mixed_out
cudaMemcpyAsync(d_mixed_out, d_agent, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
// 2. context_conv: Conv1x1 (2*ch -> ch) on joint -> GroupNorm(8, ch) -> SiLU -> ctx
float* w_ctx = engine->get_tensor(prefix + "context_conv.0.weight");
float* b_ctx = engine->get_tensor(prefix + "context_conv.0.bias");
conv2d_1x1_forward(handle, stream, d_mixed_out, w_ctx, b_ctx, d_scratch1, 2 * ch, spatial, ch);
float* g_ctx = engine->get_tensor(prefix + "context_conv.1.weight");
float* b_ctx_gn = engine->get_tensor(prefix + "context_conv.1.bias");
float* d_ctx = d_scratch2;
cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream);
fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch1, engine->get_gn_stats(), ch, spatial);
fast_groupnorm_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch1, engine->get_gn_stats(), g_ctx, b_ctx_gn, d_ctx, ch, spatial, true);
// 3. agent_proj (3x3 on ctx) -> d_scratch1
float* w_ap = engine->get_tensor(prefix + "agent_proj.weight");
float* b_ap = engine->get_tensor(prefix + "agent_proj.bias");
conv2d_3x3_forward(handle, stream, d_ctx, w_ap, b_ap, d_scratch1, d_im2col, ch, H, W, ch, 1, 1);
// 4. agent_gate (1x1 on ctx) -> d_agent_out (temporary gate buffer)
float* w_ag = engine->get_tensor(prefix + "agent_gate.weight");
float* b_ag = engine->get_tensor(prefix + "agent_gate.bias");
conv2d_1x1_forward(handle, stream, d_ctx, w_ag, b_ag, d_agent_out, ch, spatial, ch);
// f_agent_out = d_agent + d_scratch1 * sigmoid(d_agent_out)
gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>(
d_agent, d_scratch1, d_agent_out, d_agent_out, ch * spatial
);
// 5. wrist_proj (3x3 on ctx) -> d_scratch1
float* w_wp = engine->get_tensor(prefix + "wrist_proj.weight");
float* b_wp = engine->get_tensor(prefix + "wrist_proj.bias");
conv2d_3x3_forward(handle, stream, d_ctx, w_wp, b_wp, d_scratch1, d_im2col, ch, H, W, ch, 1, 1);
// 6. wrist_gate (1x1 on ctx) -> d_wrist_out (temporary gate buffer)
float* w_wg = engine->get_tensor(prefix + "wrist_gate.weight");
float* b_wg = engine->get_tensor(prefix + "wrist_gate.bias");
conv2d_1x1_forward(handle, stream, d_ctx, w_wg, b_wg, d_wrist_out, ch, spatial, ch);
// f_wrist_out = d_wrist + d_scratch1 * sigmoid(d_wrist_out)
gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>(
d_wrist, d_scratch1, d_wrist_out, d_wrist_out, ch * spatial
);
// 7. f_mixed = cat(f_agent_out, f_wrist_out)
cudaMemcpyAsync(d_mixed_out, d_agent_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
}
// -------------------------------------------------------------
// Multi-Head Attention Pooling Forward (Fused Projections & Precomputed Q)
// -------------------------------------------------------------
void attn_pool_forward(
cublasHandle_t handle,
cudaStream_t stream,
const float* d_mixed,
float* d_out_tokens_4x256,
float* d_k_proj, // [N, 256]
float* d_v_proj, // [N, 256]
float* d_k_head, // [4, N, 64]
float* d_v_head, // [4, N, 64]
float* d_scores, // [4, 4, N]
float* d_ctx_head, // [4, 4, 64]
float* d_context, // [4, 256]
int stage,
ParvulusEngine* engine,
int in_c, int H, int W
) {
int N = H * W;
float alpha = 1.0f, beta = 0.0f;
// 1. Fused K projection directly from d_mixed (eliminates in_proj GEMM & bias)
float* w_k_fused = engine->get_attn_wk_fused(stage);
float* b_k_fused = engine->get_attn_bk_fused(stage);
cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_k_fused, in_c, d_mixed, N, &beta, d_k_proj, 256);
add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_k_proj, b_k_fused, N, 256);
transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>(
d_k_proj, d_k_head, N, 4, 64
);
// 2. Fused V projection directly from d_mixed (eliminates in_proj GEMM & bias)
float* w_v_fused = engine->get_attn_wv_fused(stage);
float* b_v_fused = engine->get_attn_bv_fused(stage);
cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_v_fused, in_c, d_mixed, N, &beta, d_v_proj, 256);
add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_v_proj, b_v_fused, N, 256);
transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>(
d_v_proj, d_v_head, N, 4, 64
);
// 3. Precomputed Q head (Zero FLOPs, Zero runtime GEMM)
float* d_q_head = engine->get_attn_q_head(stage);
// 4. Batched GEMM: Scores = Q * K^T * 0.125
float scale = 0.125f;
cublasSgemmStridedBatched(
handle,
CUBLAS_OP_T, CUBLAS_OP_N,
N, 4, 64,
&scale,
d_k_head, 64, 64 * N,
d_q_head, 64, 4 * 64,
&beta,
d_scores, N, 4 * N,
4
);
// 5. Softmax over N keys for each head & query
attn_pool_softmax_kernel<<<dim3(4, 4), 256, 0, stream>>>(d_scores, N);
// 6. Context_h = Scores_h * V_h
// Scores_h: (N x 4 col-major)
// V_h: (64 x N col-major, lda = 64)
// Context_h = V_h * Scores_h: (64 x N) x (N x 4) = (64 x 4 col-major, ldc = 64)
cublasSgemmStridedBatched(
handle,
CUBLAS_OP_N, CUBLAS_OP_N,
64, 4, N,
&alpha,
d_v_head, 64, 64 * N,
d_scores, N, 4 * N,
&beta,
d_ctx_head, 64, 4 * 64,
4
);
// Transpose context from head-major [4 heads, 4 queries, 64 dims] to interleaved [4 queries, 256 dims]
transpose_head_to_interleaved_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream>>>(
d_ctx_head, d_context, 4, 4, 64
);
// 7. Out projection:
std::string prefix = "encoder.attn_poolers." + std::to_string(stage) + ".";
float* w_out = engine->get_tensor(prefix + "cross_attn.out_proj.weight");
float* b_out = engine->get_tensor(prefix + "cross_attn.out_proj.bias");
cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_out, 256, d_context, 256, &beta, d_out_tokens_4x256, 256);
add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream>>>(d_out_tokens_4x256, b_out, 4, 256);
// 8. LayerNorm(256)
float* g_norm = engine->get_tensor(prefix + "norm.weight");
float* b_norm = engine->get_tensor(prefix + "norm.bias");
layernorm_256_kernel<<<4, 256, 0, stream>>>(d_out_tokens_4x256, g_norm, b_norm, d_out_tokens_4x256, 4);
}
// -----------------------------------------------------------------------------
// Complete DualView Vision Encoder Forward Pass
// -----------------------------------------------------------------------------
void dual_view_encoder_forward(
ParvulusEngine* engine,
cublasHandle_t handle,
cudaStream_t stream,
const float* d_obs,
const float* d_cond,
float* d_z_vis,
float* d_scratch_a,
float* d_scratch_b,
float* d_scratch_c,
float* d_scratch_d,
float* d_im2col,
float* d_film_ss,
float* d_pool_k,
float* d_pool_v,
float* d_pool_k_head,
float* d_pool_v_head,
float* d_pool_q,
float* d_pool_q_head,
float* d_pool_scores,
float* d_pool_ctx_head,
float* d_pool_context
) {
const float* d_agent_raw = d_obs;
const float* d_wrist_raw = d_obs + 3 * 256 * 256;
// 1. Stem convolutions: Conv3x3 (3 -> 48, pad=1, stride=1) on 256x256
float* w_a_stem = engine->get_tensor("encoder.agent_stem.weight");
float* b_a_stem = engine->get_tensor("encoder.agent_stem.bias");
conv2d_3x3_forward(handle, stream, d_agent_raw, w_a_stem, b_a_stem, d_scratch_a, d_im2col, 3, 256, 256, 48, 1, 1);
float* w_w_stem = engine->get_tensor("encoder.wrist_stem.weight");
float* b_w_stem = engine->get_tensor("encoder.wrist_stem.bias");
conv2d_3x3_forward(handle, stream, d_wrist_raw, w_w_stem, b_w_stem, d_scratch_b, d_im2col, 3, 256, 256, 48, 1, 1);
// 2. Downsample stem 0.5x (256x256 -> 128x128):
avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_a, d_scratch_c, 48, 128, 128);
avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_b, d_scratch_d, 48, 128, 128);
int channel_mults[4] = {1, 2, 2, 4};
int in_channels[4] = {48, 48, 96, 96};
int resolutions[4] = {128, 64, 32, 16};
for (int stage = 0; stage < 4; ++stage) {
int in_ch = in_channels[stage];
int out_ch = 48 * channel_mults[stage];
int H = resolutions[stage];
int W = resolutions[stage];
std::string pfx_a = "encoder.agent_stages." + std::to_string(stage) + ".0.";
std::string pfx_w = "encoder.wrist_stages." + std::to_string(stage) + ".0.";
std::string pfx_mix = "encoder.cv_mixers." + std::to_string(stage) + ".";
std::string pfx_pool = "encoder.attn_poolers." + std::to_string(stage) + ".";
std::string pfx_down = "encoder.downsamples." + std::to_string(stage) + ".";
float* d_stage_tokens = d_z_vis + stage * 4 * 256;
if (stage % 2 == 0) {
// Inputs are in d_scratch_c (agent) and d_scratch_d (wrist)
// (a) FiLM ResBlocks -> write to d_scratch_a and d_scratch_b
film_resblock_forward(
handle, stream, d_scratch_c, d_scratch_a, d_pool_k, d_pool_v,
d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W
);
film_resblock_forward(
handle, stream, d_scratch_d, d_scratch_b, d_pool_k, d_pool_v,
d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W
);
// (b) CVMixer: inputs (a, b) -> agent_out (c), wrist_out (d), mixed_out (a)
cv_mixer_forward(
handle, stream, d_scratch_a, d_scratch_b, d_scratch_c, d_scratch_d, d_scratch_a,
d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W
);
// (c) AttnPool2d: mixed is in d_scratch_a
attn_pool_forward(
handle, stream, d_scratch_a, d_stage_tokens,
d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head,
d_pool_scores, d_pool_ctx_head, d_pool_context,
stage, engine, 2 * out_ch, H, W
);
// (d) Downsample: read (c, d) -> write to (a, b)
if (stage < 3) {
float* w_down = engine->get_tensor(pfx_down + "weight");
float* b_down = engine->get_tensor(pfx_down + "bias");
conv2d_3x3_forward(handle, stream, d_scratch_c, w_down, b_down, d_scratch_a, d_im2col, out_ch, H, W, out_ch, 2, 1);
conv2d_3x3_forward(handle, stream, d_scratch_d, w_down, b_down, d_scratch_b, d_im2col, out_ch, H, W, out_ch, 2, 1);
}
} else {
// Inputs are in d_scratch_a (agent) and d_scratch_b (wrist)
// (a) FiLM ResBlocks -> write to d_scratch_c and d_scratch_d
film_resblock_forward(
handle, stream, d_scratch_a, d_scratch_c, d_pool_k, d_pool_v,
d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W
);
film_resblock_forward(
handle, stream, d_scratch_b, d_scratch_d, d_pool_k, d_pool_v,
d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W
);
// (b) CVMixer: inputs (c, d) -> agent_out (a), wrist_out (b), mixed_out (c)
cv_mixer_forward(
handle, stream, d_scratch_c, d_scratch_d, d_scratch_a, d_scratch_b, d_scratch_c,
d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W
);
// (c) AttnPool2d: mixed is in d_scratch_c
attn_pool_forward(
handle, stream, d_scratch_c, d_stage_tokens,
d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head,
d_pool_scores, d_pool_ctx_head, d_pool_context,
stage, engine, 2 * out_ch, H, W
);
// (d) Downsample: read (a, b) -> write to (c, d)
if (stage < 3) {
float* w_down = engine->get_tensor(pfx_down + "weight");
float* b_down = engine->get_tensor(pfx_down + "bias");
conv2d_3x3_forward(handle, stream, d_scratch_a, w_down, b_down, d_scratch_c, d_im2col, out_ch, H, W, out_ch, 2, 1);
conv2d_3x3_forward(handle, stream, d_scratch_b, w_down, b_down, d_scratch_d, d_im2col, out_ch, H, W, out_ch, 2, 1);
}
}
}
}
void test_groupnorm(
const float* d_in, const float* gamma, const float* beta, float* d_out,
int C, int H, int W, bool silu, cudaStream_t stream
) {
groupnorm_2d_kernel<<<8, 256, 0, stream>>>(d_in, gamma, beta, d_out, C, H, W, silu);
}
void test_film_resblock(
ParvulusEngine* engine,
cublasHandle_t handle,
cudaStream_t stream,
const float* d_in,
float* d_out,
float* d_scratch1,
float* d_scratch2,
float* d_im2col,
float* d_film_ss,
const float* d_cond,
const std::string& prefix,
int in_c, int out_c, int H, int W
) {
film_resblock_forward(
handle, stream, d_in, d_out, d_scratch1, d_scratch2, d_im2col, d_film_ss,
d_cond, prefix, engine, in_c, out_c, H, W
);
}
void test_attn_pool(
ParvulusEngine* engine,
cublasHandle_t handle,
cudaStream_t stream,
const float* d_mixed,
float* d_out_tokens,
float* d_scratch_b,
float* d_pool_k,
float* d_pool_v,
float* d_pool_k_head,
float* d_pool_v_head,
float* d_pool_q,
float* d_pool_q_head,
float* d_pool_scores,
float* d_pool_ctx_head,
float* d_pool_context,
const std::string& prefix,
int in_c, int H, int W
) {
attn_pool_forward(
handle, stream, d_mixed, d_out_tokens,
d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head,
d_pool_scores, d_pool_ctx_head, d_pool_context,
0, engine, in_c, H, W
);
}