#include "parvulus_engine.h" #include "kernels.cuh" #include "vision_encoder.cu" #include #include #include #include #include ParvulusEngine::ParvulusEngine(int device_id) : device_id_(device_id) { cudaSetDevice(device_id_); cudaStreamCreate(&stream_); cublasCreate(&cublas_handle_); cublasSetStream(cublas_handle_, stream_); rng_.seed(42 + device_id_); allocate_scratchpads(); } ParvulusEngine::~ParvulusEngine() { free_scratchpads(); if (ode_graph_exec_) { cudaGraphExecDestroy(ode_graph_exec_); cudaGraphDestroy(ode_graph_); } if (d_weights_buffer_) cudaFree(d_weights_buffer_); if (d_text_emb_) cudaFree(d_text_emb_); if (d_action_min_) cudaFree(d_action_min_); if (d_action_max_) cudaFree(d_action_max_); cublasDestroy(cublas_handle_); cudaStreamDestroy(stream_); } void ParvulusEngine::allocate_scratchpads() { // Scratchpads for zero-allocation runtime cudaMalloc(&d_obs_, 6 * 256 * 256 * sizeof(float)); cudaMalloc(&d_z_vis_, 16 * 256 * sizeof(float)); cudaMalloc(&d_z_sigma_, 8 * 7 * sizeof(float)); cudaMalloc(&d_v_pred_, 8 * 7 * sizeof(float)); cudaMalloc(&d_cond_, 256 * sizeof(float)); cudaMalloc(&d_t_emb_, 256 * sizeof(float)); // Large vision activation buffers (384x128x128 max) cudaMalloc(&d_vis_scratch_a_, 384 * 128 * 128 * sizeof(float)); cudaMalloc(&d_vis_scratch_b_, 384 * 128 * 128 * sizeof(float)); cudaMalloc(&d_vis_scratch_c_, 384 * 128 * 128 * sizeof(float)); cudaMalloc(&d_vis_scratch_d_, 384 * 128 * 128 * sizeof(float)); cudaMalloc(&d_im2col_, 192 * 9 * 128 * 128 * sizeof(float)); cudaMalloc(&d_film_ss_, 1024 * sizeof(float)); cudaMalloc(&d_pool_k_, 16384 * 256 * sizeof(float)); cudaMalloc(&d_pool_v_, 16384 * 256 * sizeof(float)); cudaMalloc(&d_pool_k_head_, 16384 * 256 * sizeof(float)); cudaMalloc(&d_pool_v_head_, 16384 * 256 * sizeof(float)); cudaMalloc(&d_pool_q_, 4 * 256 * sizeof(float)); cudaMalloc(&d_pool_q_head_, 4 * 256 * sizeof(float)); cudaMalloc(&d_pool_scores_, 4 * 4 * 16384 * sizeof(float)); cudaMalloc(&d_pool_ctx_head_, 4 * 256 * sizeof(float)); cudaMalloc(&d_pool_context_, 4 * 256 * sizeof(float)); cudaMalloc(&d_gn_stats_, 16 * sizeof(float)); int stage_in_ch[4] = {96, 192, 192, 384}; for (int s = 0; s < 4; ++s) { int in_c = stage_in_ch[s]; cudaMalloc(&d_attn_pool_wk_fused_[s], 256 * in_c * sizeof(float)); cudaMalloc(&d_attn_pool_bk_fused_[s], 256 * sizeof(float)); cudaMalloc(&d_attn_pool_wv_fused_[s], 256 * in_c * sizeof(float)); cudaMalloc(&d_attn_pool_bv_fused_[s], 256 * sizeof(float)); cudaMalloc(&d_attn_pool_q_head_[s], 4 * 4 * 64 * sizeof(float)); } // Transformer scratchpads cudaMalloc(&d_trans_h_, 8 * 256 * sizeof(float)); cudaMalloc(&d_trans_norm_, 8 * 256 * sizeof(float)); cudaMalloc(&d_trans_qkv_, 8 * 768 * sizeof(float)); cudaMalloc(&d_trans_q_head_, 8 * 8 * 32 * sizeof(float)); cudaMalloc(&d_trans_k_head_, 8 * 16 * 32 * sizeof(float)); cudaMalloc(&d_trans_v_head_, 8 * 16 * 32 * sizeof(float)); cudaMalloc(&d_trans_scores_, 8 * 8 * 16 * sizeof(float)); cudaMalloc(&d_trans_ctx_head_, 8 * 8 * 32 * sizeof(float)); cudaMalloc(&d_trans_context_, 8 * 256 * sizeof(float)); cudaMalloc(&d_trans_out_, 8 * 256 * sizeof(float)); cudaMalloc(&d_trans_mlp_, 8 * 1024 * sizeof(float)); cudaMalloc(&d_trans_film_, 512 * sizeof(float)); cudaMalloc(&d_temp_k_, 16 * 256 * sizeof(float)); cudaMalloc(&d_temp_v_, 16 * 256 * sizeof(float)); // Precomputed Visual KV Cache scratchpads (4 layers x 8 heads x 16 keys x 32 dims) cudaMalloc(&d_cached_k_, 4 * 8 * 16 * 32 * sizeof(float)); cudaMalloc(&d_cached_v_, 4 * 8 * 16 * 32 * sizeof(float)); cudaMalloc(&d_q_proj_, 8 * 256 * sizeof(float)); cudaMalloc(&d_action_min_, 7 * sizeof(float)); cudaMalloc(&d_action_max_, 7 * sizeof(float)); cudaMalloc(&d_text_emb_, 256 * sizeof(float)); cudaMalloc(&d_raw_agent_, 256 * 256 * 3); cudaMalloc(&d_raw_wrist_, 256 * 256 * 3); cudaMalloc(&d_final_actions_, 8 * 7 * sizeof(float)); } void ParvulusEngine::free_scratchpads() { cudaFree(d_obs_); cudaFree(d_z_vis_); cudaFree(d_z_sigma_); cudaFree(d_v_pred_); cudaFree(d_cond_); cudaFree(d_t_emb_); cudaFree(d_vis_scratch_a_); cudaFree(d_vis_scratch_b_); cudaFree(d_vis_scratch_c_); cudaFree(d_vis_scratch_d_); cudaFree(d_im2col_); cudaFree(d_film_ss_); cudaFree(d_pool_k_); cudaFree(d_pool_v_); cudaFree(d_pool_k_head_); cudaFree(d_pool_v_head_); cudaFree(d_pool_q_); cudaFree(d_pool_q_head_); cudaFree(d_pool_scores_); cudaFree(d_pool_ctx_head_); cudaFree(d_pool_context_); if (d_gn_stats_) cudaFree(d_gn_stats_); for (int s = 0; s < 4; ++s) { if (d_attn_pool_wk_fused_[s]) cudaFree(d_attn_pool_wk_fused_[s]); if (d_attn_pool_bk_fused_[s]) cudaFree(d_attn_pool_bk_fused_[s]); if (d_attn_pool_wv_fused_[s]) cudaFree(d_attn_pool_wv_fused_[s]); if (d_attn_pool_bv_fused_[s]) cudaFree(d_attn_pool_bv_fused_[s]); if (d_attn_pool_q_head_[s]) cudaFree(d_attn_pool_q_head_[s]); } cudaFree(d_trans_h_); cudaFree(d_trans_norm_); cudaFree(d_trans_qkv_); cudaFree(d_trans_q_head_); cudaFree(d_trans_k_head_); cudaFree(d_trans_v_head_); cudaFree(d_trans_scores_); cudaFree(d_trans_ctx_head_); cudaFree(d_trans_context_); cudaFree(d_trans_out_); cudaFree(d_trans_mlp_); cudaFree(d_trans_film_); cudaFree(d_temp_k_); cudaFree(d_temp_v_); cudaFree(d_cached_k_); cudaFree(d_cached_v_); cudaFree(d_q_proj_); cudaFree(d_raw_agent_); cudaFree(d_raw_wrist_); cudaFree(d_final_actions_); } bool ParvulusEngine::load_weights(const std::string& weights_path) { std::ifstream file(weights_path, std::ios::binary); if (!file.is_open()) { std::cerr << "[ParvulusEngine] Failed to open weights file: " << weights_path << std::endl; return false; } uint32_t magic, version, num_tensors, header_size; file.read(reinterpret_cast(&magic), 4); file.read(reinterpret_cast(&version), 4); file.read(reinterpret_cast(&num_tensors), 4); file.read(reinterpret_cast(&header_size), 4); if (magic != 0x54524B33 && magic != 0x50415256) { std::cerr << "[ParvulusEngine] Invalid magic number in weight binary!" << std::endl; return false; } tensors_.clear(); size_t data_section_size = 0; for (uint32_t i = 0; i < num_tensors; ++i) { uint16_t name_len; file.read(reinterpret_cast(&name_len), 2); std::string name(name_len, '\0'); file.read(&name[0], name_len); uint16_t num_dims; uint32_t dims[4]; uint64_t offset, size_bytes; file.read(reinterpret_cast(&num_dims), 2); file.read(reinterpret_cast(dims), 4 * 4); file.read(reinterpret_cast(&offset), 8); file.read(reinterpret_cast(&size_bytes), 8); TensorInfo info; info.name = name; for (int d = 0; d < num_dims; ++d) info.shape.push_back(dims[d]); info.offset = offset; info.size_bytes = size_bytes; tensors_[name] = info; data_section_size = std::max(data_section_size, (size_t)(offset + size_bytes)); } total_weight_bytes_ = data_section_size; std::vector h_weights_data(data_section_size); file.read(h_weights_data.data(), data_section_size); cudaMalloc(&d_weights_buffer_, total_weight_bytes_); cudaMemcpy(d_weights_buffer_, h_weights_data.data(), total_weight_bytes_, cudaMemcpyHostToDevice); // Map device pointers for (auto& pair : tensors_) { pair.second.d_ptr = reinterpret_cast(reinterpret_cast(d_weights_buffer_) + pair.second.offset); } // Precompute / Fuse AttnPool weights and static query heads for all 4 stages int stage_in_ch[4] = {96, 192, 192, 384}; float alpha = 1.0f, beta = 0.0f; for (int s = 0; s < 4; ++s) { int in_c = stage_in_ch[s]; std::string pfx = "encoder.attn_poolers." + std::to_string(s) + "."; float* w_ip = get_tensor(pfx + "in_proj.weight"); float* b_ip = get_tensor(pfx + "in_proj.bias"); float* w_qkv = get_tensor(pfx + "cross_attn.in_proj_weight"); float* b_qkv = get_tensor(pfx + "cross_attn.in_proj_bias"); float* q_in = get_tensor(pfx + "queries"); float* w_q = w_qkv; float* b_q = b_qkv; float* w_k = w_qkv + 256 * 256; float* b_k = b_qkv + 256; float* w_v = w_qkv + 512 * 256; float* b_v = b_qkv + 512; // 1. Fused W_K: in_c x 256 col-major cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_k, 256, &beta, d_attn_pool_wk_fused_[s], in_c); // Fused b_K: W_K * b_ip + b_K cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_k, 256, b_ip, 1, &beta, d_attn_pool_bk_fused_[s], 1); add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bk_fused_[s], b_k, 256); // 2. Fused W_V: in_c x 256 col-major cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_v, 256, &beta, d_attn_pool_wv_fused_[s], in_c); // Fused b_V: W_V * b_ip + b_V cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_v, 256, b_ip, 1, &beta, d_attn_pool_bv_fused_[s], 1); add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bv_fused_[s], b_v, 256); // 3. Precompute Q_head: // q_in [4, 256] * W_Q^T + b_Q -> temp in d_pool_q_ cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_q, 256, q_in, 256, &beta, d_pool_q_, 256); add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream_>>>(d_pool_q_, b_q, 4, 256); transpose_interleaved_to_head_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream_>>>( d_pool_q_, d_attn_pool_q_head_[s], 4, 4, 64 ); } cudaStreamSynchronize(stream_); std::cout << "[ParvulusEngine] Successfully loaded " << num_tensors << " weight tensors (" << (total_weight_bytes_ / (1024.0 * 1024.0)) << " MB) into GPU VRAM." << std::endl; return true; } bool ParvulusEngine::load_task_embeddings(const std::string& embeddings_path) { std::ifstream file(embeddings_path, std::ios::binary); if (!file.is_open()) { std::cerr << "[ParvulusEngine] Failed to open task embeddings: " << embeddings_path << std::endl; return false; } uint32_t num_tasks; file.read(reinterpret_cast(&num_tasks), 4); task_embeddings_.clear(); for (uint32_t i = 0; i < num_tasks; ++i) { uint16_t str_len; file.read(reinterpret_cast(&str_len), 2); std::string task_str(str_len, '\0'); file.read(&task_str[0], str_len); std::vector emb(256); file.read(reinterpret_cast(emb.data()), 256 * sizeof(float)); task_embeddings_[task_str] = emb; } std::cout << "[ParvulusEngine] Loaded " << num_tasks << " pre-cached task language embeddings." << std::endl; return true; } bool ParvulusEngine::load_action_scale(const std::string& scale_path) { std::ifstream file(scale_path, std::ios::binary); if (!file.is_open()) { std::cerr << "[ParvulusEngine] Failed to open action scale: " << scale_path << std::endl; return false; } file.read(reinterpret_cast(h_action_min_), 7 * sizeof(float)); file.read(reinterpret_cast(h_action_max_), 7 * sizeof(float)); cudaMemcpy(d_action_min_, h_action_min_, 7 * sizeof(float), cudaMemcpyHostToDevice); cudaMemcpy(d_action_max_, h_action_max_, 7 * sizeof(float), cudaMemcpyHostToDevice); return true; } float* ParvulusEngine::get_tensor(const std::string& name) { auto it = tensors_.find(name); if (it == tensors_.end()) { std::cerr << "[ParvulusEngine] Error: Tensor not found: " << name << std::endl; return nullptr; } return it->second.d_ptr; } // ------------------------------------------------------------- // Precompute Visual Key/Value Cache (Evaluated ONCE per chunk) // ------------------------------------------------------------- void ParvulusEngine::precompute_kv_cache() { float alpha = 1.0f, beta = 0.0f; for (int layer = 0; layer < 4; ++layer) { std::string prefix = "action_decoder.layers." + std::to_string(layer) + "."; float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight"); float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias"); if (!w_c_qkv) continue; // In PyTorch MultiheadAttention in_proj_weight: shape [768, 256] // K weight: rows 256..511 (offset 256 * 256), bias: rows 256..511 (offset 256) // V weight: rows 512..767 (offset 512 * 256), bias: rows 512..767 (offset 512) float* w_k = w_c_qkv + 256 * 256; float* b_k = b_c_qkv + 256; float* w_v = w_c_qkv + 512 * 256; float* b_v = b_c_qkv + 512; float* out_k = d_cached_k_ + layer * 8 * 16 * 32; float* out_v = d_cached_v_ + layer * 8 * 16 * 32; // 1. K = z_vis * W_K^T + b_K: [16, 256] x [256, 256]^T -> [16, 256] cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_k, 256, d_z_vis_, 256, &beta, d_temp_k_, 256); add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_k_, b_k, 16, 256); // 2. V = z_vis * W_V^T + b_V: [16, 256] x [256, 256]^T -> [16, 256] cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_v, 256, d_z_vis_, 256, &beta, d_temp_v_, 256); add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_v_, b_v, 16, 256); // 3. Transpose to head-major: [16 tokens, 8 heads, 32 dims] -> [8 heads, 16 tokens, 32 dims] transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( d_temp_k_, out_k, 16, 8, 32 ); transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( d_temp_v_, out_v, 16, 8, 32 ); } } // ------------------------------------------------------------- // Timestep Conditioning MLP // ------------------------------------------------------------- void ParvulusEngine::forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond) { float alpha = 1.0f, beta = 0.0f; sinusoidal_embedding_kernel<<<1, 128, 0, stream_>>>(t, d_t_emb_, 256); float* w0 = get_tensor("action_t_mlp.mlp.0.weight"); float* b0 = get_tensor("action_t_mlp.mlp.0.bias"); cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w0, 256, d_t_emb_, 1, &beta, d_trans_norm_, 1); add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, b0, 256); silu_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_norm_, 256); float* w2 = get_tensor("action_t_mlp.mlp.2.weight"); float* b2 = get_tensor("action_t_mlp.mlp.2.bias"); cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w2, 256, d_trans_norm_, 1, &beta, d_t_emb_, 1); add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_t_emb_, b2, 256); add_residual_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_text_emb, d_t_emb_, d_cond, 256); } // ------------------------------------------------------------- // Fast KV-Cached Transformer Decoder Step // ------------------------------------------------------------- void ParvulusEngine::forward_decoder_kv( const float* d_z_sigma_8x7, const float* d_cond_256, float* d_v_pred_8x7 ) { float alpha = 1.0f, beta = 0.0f; // 1. Initial linear projection + pos_emb: h = z_sigma * W_in^T + b_in + pos_emb float* w_in = get_tensor("action_decoder.action_in_proj.weight"); float* b_in = get_tensor("action_decoder.action_in_proj.bias"); float* w_pos = get_tensor("action_decoder.pos_emb"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 7, &alpha, w_in, 7, d_z_sigma_8x7, 7, &beta, d_trans_h_, 256); add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, b_in, 8, 256); add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, w_pos, d_trans_h_, 8 * 256); float scale = 0.1767766952966369f; // 1.0 / sqrt(32) // 2. Loop over 4 Transformer Decoder Layers for (int layer = 0; layer < 4; ++layer) { std::string prefix = "action_decoder.layers." + std::to_string(layer) + "."; // ------------------------------------------------------------- // (a) Temporal Self-Attention // ------------------------------------------------------------- float* g_n1 = get_tensor(prefix + "norm1.weight"); float* b_n1 = get_tensor(prefix + "norm1.bias"); layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n1, b_n1, d_trans_norm_, 8); float* w_s_qkv = get_tensor(prefix + "self_attn.in_proj_weight"); float* b_s_qkv = get_tensor(prefix + "self_attn.in_proj_bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 768, 8, 256, &alpha, w_s_qkv, 256, d_trans_norm_, 256, &beta, d_trans_qkv_, 768); add_bias_across_tokens_kernel<<<(8 * 768 + 255) / 256, 256, 0, stream_>>>(d_trans_qkv_, b_s_qkv, 8, 768); // Transpose QKV to head-major [8 heads, 8 tokens, 32 dims] transpose_qkv_to_heads_kernel<<<8, 256, 0, stream_>>>( d_trans_qkv_, d_trans_q_head_, d_trans_k_head_, d_trans_v_head_, 8, 8, 32 ); // Scores = Q_h * K_h^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 8 keys] cublasSgemmStridedBatched( cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 8, 8, 32, &scale, d_trans_k_head_, 32, 256, d_trans_q_head_, 32, 256, &beta, d_trans_scores_, 8, 64, 8 ); // Softmax across 8 keys softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 8); // Context = Scores * V_h -> [8 heads, 8 queries, 32 dims] cublasSgemmStridedBatched( cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, 32, 8, 8, &alpha, d_trans_v_head_, 32, 256, d_trans_scores_, 8, 64, &beta, d_trans_ctx_head_, 32, 256, 8 ); // Transpose context to interleaved [8 tokens, 256 dims] transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( d_trans_ctx_head_, d_trans_context_, 8, 8, 32 ); // Out projection & residual float* w_s_out = get_tensor(prefix + "self_attn.out_proj.weight"); float* b_s_out = get_tensor(prefix + "self_attn.out_proj.bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_s_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256); add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_s_out, 8, 256); add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256); // ------------------------------------------------------------- // (b) Visual Cross-Attention with Cached K and V // ------------------------------------------------------------- float* g_n2 = get_tensor(prefix + "norm2.weight"); float* b_n2 = get_tensor(prefix + "norm2.bias"); layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n2, b_n2, d_trans_norm_, 8); float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight"); float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_qkv, 256, d_trans_norm_, 256, &beta, d_q_proj_, 256); add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_q_proj_, b_c_qkv, 8, 256); // Transpose Q to head-major [8 heads, 8 queries, 32 dims] transpose_interleaved_to_head_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( d_q_proj_, d_trans_q_head_, 8, 8, 32 ); float* cached_k = d_cached_k_ + layer * 8 * 16 * 32; float* cached_v = d_cached_v_ + layer * 8 * 16 * 32; // Scores = Q_h * K_cached^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 16 keys] cublasSgemmStridedBatched( cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 16, 8, 32, &scale, cached_k, 32, 512, d_trans_q_head_, 32, 256, &beta, d_trans_scores_, 16, 128, 8 ); // Softmax across 16 keys softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 16); // Context = Scores * V_cached -> [8 heads, 8 queries, 32 dims] cublasSgemmStridedBatched( cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, 32, 8, 16, &alpha, cached_v, 32, 512, d_trans_scores_, 16, 128, &beta, d_trans_ctx_head_, 32, 256, 8 ); // Transpose context to interleaved [8 tokens, 256 dims] transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( d_trans_ctx_head_, d_trans_context_, 8, 8, 32 ); // Out projection & residual float* w_c_out = get_tensor(prefix + "cross_attn.out_proj.weight"); float* b_c_out = get_tensor(prefix + "cross_attn.out_proj.bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256); add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_c_out, 8, 256); add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256); // ------------------------------------------------------------- // (c) FiLM Modulation & MLP // ------------------------------------------------------------- float* w_film = get_tensor(prefix + "film.weight"); float* b_film = get_tensor(prefix + "film.bias"); cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 512, &alpha, w_film, 256, d_cond_256, 1, &beta, d_trans_film_, 1); add_bias_1d_kernel<<<(512 + 255) / 256, 256, 0, stream_>>>(d_trans_film_, b_film, 512); float* g_n3 = get_tensor(prefix + "norm3.weight"); float* b_n3 = get_tensor(prefix + "norm3.bias"); layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n3, b_n3, d_trans_norm_, 8); film_modulate_256_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_film_, d_trans_norm_, 8); float* w_m1 = get_tensor(prefix + "mlp.0.weight"); float* b_m1 = get_tensor(prefix + "mlp.0.bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 1024, 8, 256, &alpha, w_m1, 256, d_trans_norm_, 256, &beta, d_trans_mlp_, 1024); add_bias_across_tokens_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, b_m1, 8, 1024); gelu_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, d_trans_mlp_, 8 * 1024); float* w_m2 = get_tensor(prefix + "mlp.2.weight"); float* b_m2 = get_tensor(prefix + "mlp.2.bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 1024, &alpha, w_m2, 1024, d_trans_mlp_, 1024, &beta, d_trans_out_, 256); add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_m2, 8, 256); add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256); } // 3. Final LayerNorm & Out Head (256 -> 7) float* g_fn = get_tensor("action_decoder.final_norm.weight"); float* b_fn = get_tensor("action_decoder.final_norm.bias"); layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_fn, b_fn, d_trans_norm_, 8); float* w_out = get_tensor("action_decoder.out_head.weight"); float* b_out = get_tensor("action_decoder.out_head.bias"); cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 7, 8, 256, &alpha, w_out, 256, d_trans_norm_, 256, &beta, d_v_pred_8x7, 7); add_bias_across_tokens_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_v_pred_8x7, b_out, 8, 7); } // ------------------------------------------------------------- // Vision DualViewEncoder Forward Pass // ------------------------------------------------------------- void ParvulusEngine::forward_encoder( const float* d_obs_6x256x256, const float* d_text_emb_256, float* d_z_vis_16x256 ) { dual_view_encoder_forward( this, cublas_handle_, stream_, d_obs_6x256x256, d_text_emb_256, d_z_vis_16x256, d_vis_scratch_a_, d_vis_scratch_b_, d_vis_scratch_c_, d_vis_scratch_d_, d_im2col_, d_film_ss_, d_pool_k_, d_pool_v_, d_pool_k_head_, d_pool_v_head_, d_pool_q_, d_pool_q_head_, d_pool_scores_, d_pool_ctx_head_, d_pool_context_ ); } // ------------------------------------------------------------- bool ParvulusEngine::register_task_embedding(const std::string& task_description, const float* emb_256) { if (!emb_256) return false; task_embeddings_[task_description] = std::vector(emb_256, emb_256 + 256); return true; } // ------------------------------------------------------------- // Complete Euler Action Flow Prediction Pipeline (Vector-First) // ------------------------------------------------------------- bool ParvulusEngine::set_task_vector(const float* h_task_vector_256) { if (!h_task_vector_256) return false; cudaMemcpyAsync(d_text_emb_, h_task_vector_256, 256 * sizeof(float), cudaMemcpyHostToDevice, stream_); has_task_vector_ = true; return true; } bool ParvulusEngine::set_task_string(const std::string& task_description) { auto it = task_embeddings_.find(task_description); if (it == task_embeddings_.end()) { return false; } return set_task_vector(it->second.data()); } bool ParvulusEngine::predict_action_chunk( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, float* h_out_actions, int num_euler_steps, const float* custom_z0 ) { if (!has_task_vector_) { std::cerr << "[ParvulusEngine] Error: Task conditioning vector has not been set! Call set_task_vector() first." << std::endl; return false; } if (!h_agent_img || !h_wrist_img || !h_out_actions) { return false; } // 1. Preprocess raw RGB frames into normalized [6, 256, 256] tensor cudaMemcpyAsync(d_raw_agent_, h_agent_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_); cudaMemcpyAsync(d_raw_wrist_, h_wrist_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_); preprocess_frames_kernel<<<(256 * 256 + 255) / 256, 256, 0, stream_>>>( d_raw_agent_, d_raw_wrist_, d_obs_, 256, 256 ); // 2. Vision Forward Pass (Evaluated ONCE per chunk) forward_encoder(d_obs_, d_text_emb_, d_z_vis_); // 3. Precompute Visual KV Cache (Evaluated ONCE per chunk) precompute_kv_cache(); // 4. Sample base noise z0 if (custom_z0 != nullptr) { cudaMemcpyAsync(d_z_sigma_, custom_z0, 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_); } else { std::vector h_z0(8 * 7); for (int i = 0; i < 8 * 7; ++i) h_z0[i] = norm_dist_(rng_); cudaMemcpyAsync(d_z_sigma_, h_z0.data(), 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_); } // 5. Euler ODE Integration Loop with KV Cache float dt = 1.0f / (float)num_euler_steps; for (int step = 0; step < num_euler_steps; ++step) { float t = (float)step * dt; forward_timestep_mlp(t, d_text_emb_, d_cond_); forward_decoder_kv(d_z_sigma_, d_cond_, d_v_pred_); euler_step_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, d_v_pred_, dt, 8 * 7); } clamp_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, -1.0f, 1.0f, 8 * 7); // 6. Denormalize actions to unnormalized robot space denormalize_actions_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>( d_z_sigma_, d_action_min_, d_action_max_, d_final_actions_, 8, 7 ); cudaMemcpyAsync(h_out_actions, d_final_actions_, 8 * 7 * sizeof(float), cudaMemcpyDeviceToHost, stream_); cudaStreamSynchronize(stream_); return true; } bool ParvulusEngine::predict_action_chunk_with_vector( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, const float* h_task_vector_256, float* h_out_actions, int num_euler_steps, const float* custom_z0 ) { if (!set_task_vector(h_task_vector_256)) return false; return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0); } bool ParvulusEngine::predict_action_chunk( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, const std::string& task_description, float* h_out_actions, int num_euler_steps, const float* custom_z0 ) { if (!set_task_string(task_description)) return false; return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0); }