Parvulus / c_engine /parvulus_engine.cu
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
29.9 kB
#include "parvulus_engine.h"
#include "kernels.cuh"
#include "vision_encoder.cu"
#include <iostream>
#include <fstream>
#include <cstring>
#include <cmath>
#include <random>
ParvulusEngine::ParvulusEngine(int device_id) : device_id_(device_id) {
cudaSetDevice(device_id_);
cudaStreamCreate(&stream_);
cublasCreate(&cublas_handle_);
cublasSetStream(cublas_handle_, stream_);
rng_.seed(42 + device_id_);
allocate_scratchpads();
}
ParvulusEngine::~ParvulusEngine() {
free_scratchpads();
if (ode_graph_exec_) {
cudaGraphExecDestroy(ode_graph_exec_);
cudaGraphDestroy(ode_graph_);
}
if (d_weights_buffer_) cudaFree(d_weights_buffer_);
if (d_text_emb_) cudaFree(d_text_emb_);
if (d_action_min_) cudaFree(d_action_min_);
if (d_action_max_) cudaFree(d_action_max_);
cublasDestroy(cublas_handle_);
cudaStreamDestroy(stream_);
}
void ParvulusEngine::allocate_scratchpads() {
// Scratchpads for zero-allocation runtime
cudaMalloc(&d_obs_, 6 * 256 * 256 * sizeof(float));
cudaMalloc(&d_z_vis_, 16 * 256 * sizeof(float));
cudaMalloc(&d_z_sigma_, 8 * 7 * sizeof(float));
cudaMalloc(&d_v_pred_, 8 * 7 * sizeof(float));
cudaMalloc(&d_cond_, 256 * sizeof(float));
cudaMalloc(&d_t_emb_, 256 * sizeof(float));
// Large vision activation buffers (384x128x128 max)
cudaMalloc(&d_vis_scratch_a_, 384 * 128 * 128 * sizeof(float));
cudaMalloc(&d_vis_scratch_b_, 384 * 128 * 128 * sizeof(float));
cudaMalloc(&d_vis_scratch_c_, 384 * 128 * 128 * sizeof(float));
cudaMalloc(&d_vis_scratch_d_, 384 * 128 * 128 * sizeof(float));
cudaMalloc(&d_im2col_, 192 * 9 * 128 * 128 * sizeof(float));
cudaMalloc(&d_film_ss_, 1024 * sizeof(float));
cudaMalloc(&d_pool_k_, 16384 * 256 * sizeof(float));
cudaMalloc(&d_pool_v_, 16384 * 256 * sizeof(float));
cudaMalloc(&d_pool_k_head_, 16384 * 256 * sizeof(float));
cudaMalloc(&d_pool_v_head_, 16384 * 256 * sizeof(float));
cudaMalloc(&d_pool_q_, 4 * 256 * sizeof(float));
cudaMalloc(&d_pool_q_head_, 4 * 256 * sizeof(float));
cudaMalloc(&d_pool_scores_, 4 * 4 * 16384 * sizeof(float));
cudaMalloc(&d_pool_ctx_head_, 4 * 256 * sizeof(float));
cudaMalloc(&d_pool_context_, 4 * 256 * sizeof(float));
cudaMalloc(&d_gn_stats_, 16 * sizeof(float));
int stage_in_ch[4] = {96, 192, 192, 384};
for (int s = 0; s < 4; ++s) {
int in_c = stage_in_ch[s];
cudaMalloc(&d_attn_pool_wk_fused_[s], 256 * in_c * sizeof(float));
cudaMalloc(&d_attn_pool_bk_fused_[s], 256 * sizeof(float));
cudaMalloc(&d_attn_pool_wv_fused_[s], 256 * in_c * sizeof(float));
cudaMalloc(&d_attn_pool_bv_fused_[s], 256 * sizeof(float));
cudaMalloc(&d_attn_pool_q_head_[s], 4 * 4 * 64 * sizeof(float));
}
// Transformer scratchpads
cudaMalloc(&d_trans_h_, 8 * 256 * sizeof(float));
cudaMalloc(&d_trans_norm_, 8 * 256 * sizeof(float));
cudaMalloc(&d_trans_qkv_, 8 * 768 * sizeof(float));
cudaMalloc(&d_trans_q_head_, 8 * 8 * 32 * sizeof(float));
cudaMalloc(&d_trans_k_head_, 8 * 16 * 32 * sizeof(float));
cudaMalloc(&d_trans_v_head_, 8 * 16 * 32 * sizeof(float));
cudaMalloc(&d_trans_scores_, 8 * 8 * 16 * sizeof(float));
cudaMalloc(&d_trans_ctx_head_, 8 * 8 * 32 * sizeof(float));
cudaMalloc(&d_trans_context_, 8 * 256 * sizeof(float));
cudaMalloc(&d_trans_out_, 8 * 256 * sizeof(float));
cudaMalloc(&d_trans_mlp_, 8 * 1024 * sizeof(float));
cudaMalloc(&d_trans_film_, 512 * sizeof(float));
cudaMalloc(&d_temp_k_, 16 * 256 * sizeof(float));
cudaMalloc(&d_temp_v_, 16 * 256 * sizeof(float));
// Precomputed Visual KV Cache scratchpads (4 layers x 8 heads x 16 keys x 32 dims)
cudaMalloc(&d_cached_k_, 4 * 8 * 16 * 32 * sizeof(float));
cudaMalloc(&d_cached_v_, 4 * 8 * 16 * 32 * sizeof(float));
cudaMalloc(&d_q_proj_, 8 * 256 * sizeof(float));
cudaMalloc(&d_action_min_, 7 * sizeof(float));
cudaMalloc(&d_action_max_, 7 * sizeof(float));
cudaMalloc(&d_text_emb_, 256 * sizeof(float));
cudaMalloc(&d_raw_agent_, 256 * 256 * 3);
cudaMalloc(&d_raw_wrist_, 256 * 256 * 3);
cudaMalloc(&d_final_actions_, 8 * 7 * sizeof(float));
}
void ParvulusEngine::free_scratchpads() {
cudaFree(d_obs_);
cudaFree(d_z_vis_);
cudaFree(d_z_sigma_);
cudaFree(d_v_pred_);
cudaFree(d_cond_);
cudaFree(d_t_emb_);
cudaFree(d_vis_scratch_a_);
cudaFree(d_vis_scratch_b_);
cudaFree(d_vis_scratch_c_);
cudaFree(d_vis_scratch_d_);
cudaFree(d_im2col_);
cudaFree(d_film_ss_);
cudaFree(d_pool_k_);
cudaFree(d_pool_v_);
cudaFree(d_pool_k_head_);
cudaFree(d_pool_v_head_);
cudaFree(d_pool_q_);
cudaFree(d_pool_q_head_);
cudaFree(d_pool_scores_);
cudaFree(d_pool_ctx_head_);
cudaFree(d_pool_context_);
if (d_gn_stats_) cudaFree(d_gn_stats_);
for (int s = 0; s < 4; ++s) {
if (d_attn_pool_wk_fused_[s]) cudaFree(d_attn_pool_wk_fused_[s]);
if (d_attn_pool_bk_fused_[s]) cudaFree(d_attn_pool_bk_fused_[s]);
if (d_attn_pool_wv_fused_[s]) cudaFree(d_attn_pool_wv_fused_[s]);
if (d_attn_pool_bv_fused_[s]) cudaFree(d_attn_pool_bv_fused_[s]);
if (d_attn_pool_q_head_[s]) cudaFree(d_attn_pool_q_head_[s]);
}
cudaFree(d_trans_h_);
cudaFree(d_trans_norm_);
cudaFree(d_trans_qkv_);
cudaFree(d_trans_q_head_);
cudaFree(d_trans_k_head_);
cudaFree(d_trans_v_head_);
cudaFree(d_trans_scores_);
cudaFree(d_trans_ctx_head_);
cudaFree(d_trans_context_);
cudaFree(d_trans_out_);
cudaFree(d_trans_mlp_);
cudaFree(d_trans_film_);
cudaFree(d_temp_k_);
cudaFree(d_temp_v_);
cudaFree(d_cached_k_);
cudaFree(d_cached_v_);
cudaFree(d_q_proj_);
cudaFree(d_raw_agent_);
cudaFree(d_raw_wrist_);
cudaFree(d_final_actions_);
}
bool ParvulusEngine::load_weights(const std::string& weights_path) {
std::ifstream file(weights_path, std::ios::binary);
if (!file.is_open()) {
std::cerr << "[ParvulusEngine] Failed to open weights file: " << weights_path << std::endl;
return false;
}
uint32_t magic, version, num_tensors, header_size;
file.read(reinterpret_cast<char*>(&magic), 4);
file.read(reinterpret_cast<char*>(&version), 4);
file.read(reinterpret_cast<char*>(&num_tensors), 4);
file.read(reinterpret_cast<char*>(&header_size), 4);
if (magic != 0x54524B33 && magic != 0x50415256) {
std::cerr << "[ParvulusEngine] Invalid magic number in weight binary!" << std::endl;
return false;
}
tensors_.clear();
size_t data_section_size = 0;
for (uint32_t i = 0; i < num_tensors; ++i) {
uint16_t name_len;
file.read(reinterpret_cast<char*>(&name_len), 2);
std::string name(name_len, '\0');
file.read(&name[0], name_len);
uint16_t num_dims;
uint32_t dims[4];
uint64_t offset, size_bytes;
file.read(reinterpret_cast<char*>(&num_dims), 2);
file.read(reinterpret_cast<char*>(dims), 4 * 4);
file.read(reinterpret_cast<char*>(&offset), 8);
file.read(reinterpret_cast<char*>(&size_bytes), 8);
TensorInfo info;
info.name = name;
for (int d = 0; d < num_dims; ++d) info.shape.push_back(dims[d]);
info.offset = offset;
info.size_bytes = size_bytes;
tensors_[name] = info;
data_section_size = std::max(data_section_size, (size_t)(offset + size_bytes));
}
total_weight_bytes_ = data_section_size;
std::vector<char> h_weights_data(data_section_size);
file.read(h_weights_data.data(), data_section_size);
cudaMalloc(&d_weights_buffer_, total_weight_bytes_);
cudaMemcpy(d_weights_buffer_, h_weights_data.data(), total_weight_bytes_, cudaMemcpyHostToDevice);
// Map device pointers
for (auto& pair : tensors_) {
pair.second.d_ptr = reinterpret_cast<float*>(reinterpret_cast<char*>(d_weights_buffer_) + pair.second.offset);
}
// Precompute / Fuse AttnPool weights and static query heads for all 4 stages
int stage_in_ch[4] = {96, 192, 192, 384};
float alpha = 1.0f, beta = 0.0f;
for (int s = 0; s < 4; ++s) {
int in_c = stage_in_ch[s];
std::string pfx = "encoder.attn_poolers." + std::to_string(s) + ".";
float* w_ip = get_tensor(pfx + "in_proj.weight");
float* b_ip = get_tensor(pfx + "in_proj.bias");
float* w_qkv = get_tensor(pfx + "cross_attn.in_proj_weight");
float* b_qkv = get_tensor(pfx + "cross_attn.in_proj_bias");
float* q_in = get_tensor(pfx + "queries");
float* w_q = w_qkv;
float* b_q = b_qkv;
float* w_k = w_qkv + 256 * 256;
float* b_k = b_qkv + 256;
float* w_v = w_qkv + 512 * 256;
float* b_v = b_qkv + 512;
// 1. Fused W_K: in_c x 256 col-major
cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_k, 256, &beta, d_attn_pool_wk_fused_[s], in_c);
// Fused b_K: W_K * b_ip + b_K
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_k, 256, b_ip, 1, &beta, d_attn_pool_bk_fused_[s], 1);
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bk_fused_[s], b_k, 256);
// 2. Fused W_V: in_c x 256 col-major
cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_v, 256, &beta, d_attn_pool_wv_fused_[s], in_c);
// Fused b_V: W_V * b_ip + b_V
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_v, 256, b_ip, 1, &beta, d_attn_pool_bv_fused_[s], 1);
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bv_fused_[s], b_v, 256);
// 3. Precompute Q_head:
// q_in [4, 256] * W_Q^T + b_Q -> temp in d_pool_q_
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_q, 256, q_in, 256, &beta, d_pool_q_, 256);
add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream_>>>(d_pool_q_, b_q, 4, 256);
transpose_interleaved_to_head_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream_>>>(
d_pool_q_, d_attn_pool_q_head_[s], 4, 4, 64
);
}
cudaStreamSynchronize(stream_);
std::cout << "[ParvulusEngine] Successfully loaded " << num_tensors << " weight tensors ("
<< (total_weight_bytes_ / (1024.0 * 1024.0)) << " MB) into GPU VRAM." << std::endl;
return true;
}
bool ParvulusEngine::load_task_embeddings(const std::string& embeddings_path) {
std::ifstream file(embeddings_path, std::ios::binary);
if (!file.is_open()) {
std::cerr << "[ParvulusEngine] Failed to open task embeddings: " << embeddings_path << std::endl;
return false;
}
uint32_t num_tasks;
file.read(reinterpret_cast<char*>(&num_tasks), 4);
task_embeddings_.clear();
for (uint32_t i = 0; i < num_tasks; ++i) {
uint16_t str_len;
file.read(reinterpret_cast<char*>(&str_len), 2);
std::string task_str(str_len, '\0');
file.read(&task_str[0], str_len);
std::vector<float> emb(256);
file.read(reinterpret_cast<char*>(emb.data()), 256 * sizeof(float));
task_embeddings_[task_str] = emb;
}
std::cout << "[ParvulusEngine] Loaded " << num_tasks << " pre-cached task language embeddings." << std::endl;
return true;
}
bool ParvulusEngine::load_action_scale(const std::string& scale_path) {
std::ifstream file(scale_path, std::ios::binary);
if (!file.is_open()) {
std::cerr << "[ParvulusEngine] Failed to open action scale: " << scale_path << std::endl;
return false;
}
file.read(reinterpret_cast<char*>(h_action_min_), 7 * sizeof(float));
file.read(reinterpret_cast<char*>(h_action_max_), 7 * sizeof(float));
cudaMemcpy(d_action_min_, h_action_min_, 7 * sizeof(float), cudaMemcpyHostToDevice);
cudaMemcpy(d_action_max_, h_action_max_, 7 * sizeof(float), cudaMemcpyHostToDevice);
return true;
}
float* ParvulusEngine::get_tensor(const std::string& name) {
auto it = tensors_.find(name);
if (it == tensors_.end()) {
std::cerr << "[ParvulusEngine] Error: Tensor not found: " << name << std::endl;
return nullptr;
}
return it->second.d_ptr;
}
// -------------------------------------------------------------
// Precompute Visual Key/Value Cache (Evaluated ONCE per chunk)
// -------------------------------------------------------------
void ParvulusEngine::precompute_kv_cache() {
float alpha = 1.0f, beta = 0.0f;
for (int layer = 0; layer < 4; ++layer) {
std::string prefix = "action_decoder.layers." + std::to_string(layer) + ".";
float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight");
float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias");
if (!w_c_qkv) continue;
// In PyTorch MultiheadAttention in_proj_weight: shape [768, 256]
// K weight: rows 256..511 (offset 256 * 256), bias: rows 256..511 (offset 256)
// V weight: rows 512..767 (offset 512 * 256), bias: rows 512..767 (offset 512)
float* w_k = w_c_qkv + 256 * 256;
float* b_k = b_c_qkv + 256;
float* w_v = w_c_qkv + 512 * 256;
float* b_v = b_c_qkv + 512;
float* out_k = d_cached_k_ + layer * 8 * 16 * 32;
float* out_v = d_cached_v_ + layer * 8 * 16 * 32;
// 1. K = z_vis * W_K^T + b_K: [16, 256] x [256, 256]^T -> [16, 256]
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_k, 256, d_z_vis_, 256, &beta, d_temp_k_, 256);
add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_k_, b_k, 16, 256);
// 2. V = z_vis * W_V^T + b_V: [16, 256] x [256, 256]^T -> [16, 256]
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_v, 256, d_z_vis_, 256, &beta, d_temp_v_, 256);
add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_v_, b_v, 16, 256);
// 3. Transpose to head-major: [16 tokens, 8 heads, 32 dims] -> [8 heads, 16 tokens, 32 dims]
transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
d_temp_k_, out_k, 16, 8, 32
);
transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
d_temp_v_, out_v, 16, 8, 32
);
}
}
// -------------------------------------------------------------
// Timestep Conditioning MLP
// -------------------------------------------------------------
void ParvulusEngine::forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond) {
float alpha = 1.0f, beta = 0.0f;
sinusoidal_embedding_kernel<<<1, 128, 0, stream_>>>(t, d_t_emb_, 256);
float* w0 = get_tensor("action_t_mlp.mlp.0.weight");
float* b0 = get_tensor("action_t_mlp.mlp.0.bias");
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w0, 256, d_t_emb_, 1, &beta, d_trans_norm_, 1);
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, b0, 256);
silu_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_norm_, 256);
float* w2 = get_tensor("action_t_mlp.mlp.2.weight");
float* b2 = get_tensor("action_t_mlp.mlp.2.bias");
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w2, 256, d_trans_norm_, 1, &beta, d_t_emb_, 1);
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_t_emb_, b2, 256);
add_residual_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_text_emb, d_t_emb_, d_cond, 256);
}
// -------------------------------------------------------------
// Fast KV-Cached Transformer Decoder Step
// -------------------------------------------------------------
void ParvulusEngine::forward_decoder_kv(
const float* d_z_sigma_8x7,
const float* d_cond_256,
float* d_v_pred_8x7
) {
float alpha = 1.0f, beta = 0.0f;
// 1. Initial linear projection + pos_emb: h = z_sigma * W_in^T + b_in + pos_emb
float* w_in = get_tensor("action_decoder.action_in_proj.weight");
float* b_in = get_tensor("action_decoder.action_in_proj.bias");
float* w_pos = get_tensor("action_decoder.pos_emb");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 7, &alpha, w_in, 7, d_z_sigma_8x7, 7, &beta, d_trans_h_, 256);
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, b_in, 8, 256);
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, w_pos, d_trans_h_, 8 * 256);
float scale = 0.1767766952966369f; // 1.0 / sqrt(32)
// 2. Loop over 4 Transformer Decoder Layers
for (int layer = 0; layer < 4; ++layer) {
std::string prefix = "action_decoder.layers." + std::to_string(layer) + ".";
// -------------------------------------------------------------
// (a) Temporal Self-Attention
// -------------------------------------------------------------
float* g_n1 = get_tensor(prefix + "norm1.weight");
float* b_n1 = get_tensor(prefix + "norm1.bias");
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n1, b_n1, d_trans_norm_, 8);
float* w_s_qkv = get_tensor(prefix + "self_attn.in_proj_weight");
float* b_s_qkv = get_tensor(prefix + "self_attn.in_proj_bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 768, 8, 256, &alpha, w_s_qkv, 256, d_trans_norm_, 256, &beta, d_trans_qkv_, 768);
add_bias_across_tokens_kernel<<<(8 * 768 + 255) / 256, 256, 0, stream_>>>(d_trans_qkv_, b_s_qkv, 8, 768);
// Transpose QKV to head-major [8 heads, 8 tokens, 32 dims]
transpose_qkv_to_heads_kernel<<<8, 256, 0, stream_>>>(
d_trans_qkv_, d_trans_q_head_, d_trans_k_head_, d_trans_v_head_, 8, 8, 32
);
// Scores = Q_h * K_h^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 8 keys]
cublasSgemmStridedBatched(
cublas_handle_,
CUBLAS_OP_T, CUBLAS_OP_N,
8, 8, 32,
&scale,
d_trans_k_head_, 32, 256,
d_trans_q_head_, 32, 256,
&beta,
d_trans_scores_, 8, 64,
8
);
// Softmax across 8 keys
softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 8);
// Context = Scores * V_h -> [8 heads, 8 queries, 32 dims]
cublasSgemmStridedBatched(
cublas_handle_,
CUBLAS_OP_N, CUBLAS_OP_N,
32, 8, 8,
&alpha,
d_trans_v_head_, 32, 256,
d_trans_scores_, 8, 64,
&beta,
d_trans_ctx_head_, 32, 256,
8
);
// Transpose context to interleaved [8 tokens, 256 dims]
transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
d_trans_ctx_head_, d_trans_context_, 8, 8, 32
);
// Out projection & residual
float* w_s_out = get_tensor(prefix + "self_attn.out_proj.weight");
float* b_s_out = get_tensor(prefix + "self_attn.out_proj.bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_s_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256);
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_s_out, 8, 256);
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256);
// -------------------------------------------------------------
// (b) Visual Cross-Attention with Cached K and V
// -------------------------------------------------------------
float* g_n2 = get_tensor(prefix + "norm2.weight");
float* b_n2 = get_tensor(prefix + "norm2.bias");
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n2, b_n2, d_trans_norm_, 8);
float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight");
float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_qkv, 256, d_trans_norm_, 256, &beta, d_q_proj_, 256);
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_q_proj_, b_c_qkv, 8, 256);
// Transpose Q to head-major [8 heads, 8 queries, 32 dims]
transpose_interleaved_to_head_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
d_q_proj_, d_trans_q_head_, 8, 8, 32
);
float* cached_k = d_cached_k_ + layer * 8 * 16 * 32;
float* cached_v = d_cached_v_ + layer * 8 * 16 * 32;
// Scores = Q_h * K_cached^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 16 keys]
cublasSgemmStridedBatched(
cublas_handle_,
CUBLAS_OP_T, CUBLAS_OP_N,
16, 8, 32,
&scale,
cached_k, 32, 512,
d_trans_q_head_, 32, 256,
&beta,
d_trans_scores_, 16, 128,
8
);
// Softmax across 16 keys
softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 16);
// Context = Scores * V_cached -> [8 heads, 8 queries, 32 dims]
cublasSgemmStridedBatched(
cublas_handle_,
CUBLAS_OP_N, CUBLAS_OP_N,
32, 8, 16,
&alpha,
cached_v, 32, 512,
d_trans_scores_, 16, 128,
&beta,
d_trans_ctx_head_, 32, 256,
8
);
// Transpose context to interleaved [8 tokens, 256 dims]
transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
d_trans_ctx_head_, d_trans_context_, 8, 8, 32
);
// Out projection & residual
float* w_c_out = get_tensor(prefix + "cross_attn.out_proj.weight");
float* b_c_out = get_tensor(prefix + "cross_attn.out_proj.bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256);
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_c_out, 8, 256);
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256);
// -------------------------------------------------------------
// (c) FiLM Modulation & MLP
// -------------------------------------------------------------
float* w_film = get_tensor(prefix + "film.weight");
float* b_film = get_tensor(prefix + "film.bias");
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 512, &alpha, w_film, 256, d_cond_256, 1, &beta, d_trans_film_, 1);
add_bias_1d_kernel<<<(512 + 255) / 256, 256, 0, stream_>>>(d_trans_film_, b_film, 512);
float* g_n3 = get_tensor(prefix + "norm3.weight");
float* b_n3 = get_tensor(prefix + "norm3.bias");
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n3, b_n3, d_trans_norm_, 8);
film_modulate_256_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_film_, d_trans_norm_, 8);
float* w_m1 = get_tensor(prefix + "mlp.0.weight");
float* b_m1 = get_tensor(prefix + "mlp.0.bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 1024, 8, 256, &alpha, w_m1, 256, d_trans_norm_, 256, &beta, d_trans_mlp_, 1024);
add_bias_across_tokens_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, b_m1, 8, 1024);
gelu_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, d_trans_mlp_, 8 * 1024);
float* w_m2 = get_tensor(prefix + "mlp.2.weight");
float* b_m2 = get_tensor(prefix + "mlp.2.bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 1024, &alpha, w_m2, 1024, d_trans_mlp_, 1024, &beta, d_trans_out_, 256);
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_m2, 8, 256);
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256);
}
// 3. Final LayerNorm & Out Head (256 -> 7)
float* g_fn = get_tensor("action_decoder.final_norm.weight");
float* b_fn = get_tensor("action_decoder.final_norm.bias");
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_fn, b_fn, d_trans_norm_, 8);
float* w_out = get_tensor("action_decoder.out_head.weight");
float* b_out = get_tensor("action_decoder.out_head.bias");
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 7, 8, 256, &alpha, w_out, 256, d_trans_norm_, 256, &beta, d_v_pred_8x7, 7);
add_bias_across_tokens_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_v_pred_8x7, b_out, 8, 7);
}
// -------------------------------------------------------------
// Vision DualViewEncoder Forward Pass
// -------------------------------------------------------------
void ParvulusEngine::forward_encoder(
const float* d_obs_6x256x256,
const float* d_text_emb_256,
float* d_z_vis_16x256
) {
dual_view_encoder_forward(
this,
cublas_handle_,
stream_,
d_obs_6x256x256,
d_text_emb_256,
d_z_vis_16x256,
d_vis_scratch_a_,
d_vis_scratch_b_,
d_vis_scratch_c_,
d_vis_scratch_d_,
d_im2col_,
d_film_ss_,
d_pool_k_,
d_pool_v_,
d_pool_k_head_,
d_pool_v_head_,
d_pool_q_,
d_pool_q_head_,
d_pool_scores_,
d_pool_ctx_head_,
d_pool_context_
);
}
// -------------------------------------------------------------
bool ParvulusEngine::register_task_embedding(const std::string& task_description, const float* emb_256) {
if (!emb_256) return false;
task_embeddings_[task_description] = std::vector<float>(emb_256, emb_256 + 256);
return true;
}
// -------------------------------------------------------------
// Complete Euler Action Flow Prediction Pipeline (Vector-First)
// -------------------------------------------------------------
bool ParvulusEngine::set_task_vector(const float* h_task_vector_256) {
if (!h_task_vector_256) return false;
cudaMemcpyAsync(d_text_emb_, h_task_vector_256, 256 * sizeof(float), cudaMemcpyHostToDevice, stream_);
has_task_vector_ = true;
return true;
}
bool ParvulusEngine::set_task_string(const std::string& task_description) {
auto it = task_embeddings_.find(task_description);
if (it == task_embeddings_.end()) {
return false;
}
return set_task_vector(it->second.data());
}
bool ParvulusEngine::predict_action_chunk(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
float* h_out_actions,
int num_euler_steps,
const float* custom_z0
) {
if (!has_task_vector_) {
std::cerr << "[ParvulusEngine] Error: Task conditioning vector has not been set! Call set_task_vector() first." << std::endl;
return false;
}
if (!h_agent_img || !h_wrist_img || !h_out_actions) {
return false;
}
// 1. Preprocess raw RGB frames into normalized [6, 256, 256] tensor
cudaMemcpyAsync(d_raw_agent_, h_agent_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_);
cudaMemcpyAsync(d_raw_wrist_, h_wrist_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_);
preprocess_frames_kernel<<<(256 * 256 + 255) / 256, 256, 0, stream_>>>(
d_raw_agent_, d_raw_wrist_, d_obs_, 256, 256
);
// 2. Vision Forward Pass (Evaluated ONCE per chunk)
forward_encoder(d_obs_, d_text_emb_, d_z_vis_);
// 3. Precompute Visual KV Cache (Evaluated ONCE per chunk)
precompute_kv_cache();
// 4. Sample base noise z0
if (custom_z0 != nullptr) {
cudaMemcpyAsync(d_z_sigma_, custom_z0, 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_);
} else {
std::vector<float> h_z0(8 * 7);
for (int i = 0; i < 8 * 7; ++i) h_z0[i] = norm_dist_(rng_);
cudaMemcpyAsync(d_z_sigma_, h_z0.data(), 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_);
}
// 5. Euler ODE Integration Loop with KV Cache
float dt = 1.0f / (float)num_euler_steps;
for (int step = 0; step < num_euler_steps; ++step) {
float t = (float)step * dt;
forward_timestep_mlp(t, d_text_emb_, d_cond_);
forward_decoder_kv(d_z_sigma_, d_cond_, d_v_pred_);
euler_step_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, d_v_pred_, dt, 8 * 7);
}
clamp_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, -1.0f, 1.0f, 8 * 7);
// 6. Denormalize actions to unnormalized robot space
denormalize_actions_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(
d_z_sigma_, d_action_min_, d_action_max_, d_final_actions_, 8, 7
);
cudaMemcpyAsync(h_out_actions, d_final_actions_, 8 * 7 * sizeof(float), cudaMemcpyDeviceToHost, stream_);
cudaStreamSynchronize(stream_);
return true;
}
bool ParvulusEngine::predict_action_chunk_with_vector(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
const float* h_task_vector_256,
float* h_out_actions,
int num_euler_steps,
const float* custom_z0
) {
if (!set_task_vector(h_task_vector_256)) return false;
return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0);
}
bool ParvulusEngine::predict_action_chunk(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
const std::string& task_description,
float* h_out_actions,
int num_euler_steps,
const float* custom_z0
) {
if (!set_task_string(task_description)) return false;
return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0);
}