Download c_engine/parvulus_engine.cu from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 29.9 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/parvulus_engine.cu
- Command line
-
hf download hf://aasivas/Parvulus/c_engine/parvulus_engine.cu
-
curl -L -o parvulus_engine.cu https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/parvulus_engine.cu
29.9 kB
| #include "parvulus_engine.h" | |
| #include "kernels.cuh" | |
| #include "vision_encoder.cu" | |
| #include <iostream> | |
| #include <fstream> | |
| #include <cstring> | |
| #include <cmath> | |
| #include <random> | |
| ParvulusEngine::ParvulusEngine(int device_id) : device_id_(device_id) { | |
| cudaSetDevice(device_id_); | |
| cudaStreamCreate(&stream_); | |
| cublasCreate(&cublas_handle_); | |
| cublasSetStream(cublas_handle_, stream_); | |
| rng_.seed(42 + device_id_); | |
| allocate_scratchpads(); | |
| } | |
| ParvulusEngine::~ParvulusEngine() { | |
| free_scratchpads(); | |
| if (ode_graph_exec_) { | |
| cudaGraphExecDestroy(ode_graph_exec_); | |
| cudaGraphDestroy(ode_graph_); | |
| } | |
| if (d_weights_buffer_) cudaFree(d_weights_buffer_); | |
| if (d_text_emb_) cudaFree(d_text_emb_); | |
| if (d_action_min_) cudaFree(d_action_min_); | |
| if (d_action_max_) cudaFree(d_action_max_); | |
| cublasDestroy(cublas_handle_); | |
| cudaStreamDestroy(stream_); | |
| } | |
| void ParvulusEngine::allocate_scratchpads() { | |
| // Scratchpads for zero-allocation runtime | |
| cudaMalloc(&d_obs_, 6 * 256 * 256 * sizeof(float)); | |
| cudaMalloc(&d_z_vis_, 16 * 256 * sizeof(float)); | |
| cudaMalloc(&d_z_sigma_, 8 * 7 * sizeof(float)); | |
| cudaMalloc(&d_v_pred_, 8 * 7 * sizeof(float)); | |
| cudaMalloc(&d_cond_, 256 * sizeof(float)); | |
| cudaMalloc(&d_t_emb_, 256 * sizeof(float)); | |
| // Large vision activation buffers (384x128x128 max) | |
| cudaMalloc(&d_vis_scratch_a_, 384 * 128 * 128 * sizeof(float)); | |
| cudaMalloc(&d_vis_scratch_b_, 384 * 128 * 128 * sizeof(float)); | |
| cudaMalloc(&d_vis_scratch_c_, 384 * 128 * 128 * sizeof(float)); | |
| cudaMalloc(&d_vis_scratch_d_, 384 * 128 * 128 * sizeof(float)); | |
| cudaMalloc(&d_im2col_, 192 * 9 * 128 * 128 * sizeof(float)); | |
| cudaMalloc(&d_film_ss_, 1024 * sizeof(float)); | |
| cudaMalloc(&d_pool_k_, 16384 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_v_, 16384 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_k_head_, 16384 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_v_head_, 16384 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_q_, 4 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_q_head_, 4 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_scores_, 4 * 4 * 16384 * sizeof(float)); | |
| cudaMalloc(&d_pool_ctx_head_, 4 * 256 * sizeof(float)); | |
| cudaMalloc(&d_pool_context_, 4 * 256 * sizeof(float)); | |
| cudaMalloc(&d_gn_stats_, 16 * sizeof(float)); | |
| int stage_in_ch[4] = {96, 192, 192, 384}; | |
| for (int s = 0; s < 4; ++s) { | |
| int in_c = stage_in_ch[s]; | |
| cudaMalloc(&d_attn_pool_wk_fused_[s], 256 * in_c * sizeof(float)); | |
| cudaMalloc(&d_attn_pool_bk_fused_[s], 256 * sizeof(float)); | |
| cudaMalloc(&d_attn_pool_wv_fused_[s], 256 * in_c * sizeof(float)); | |
| cudaMalloc(&d_attn_pool_bv_fused_[s], 256 * sizeof(float)); | |
| cudaMalloc(&d_attn_pool_q_head_[s], 4 * 4 * 64 * sizeof(float)); | |
| } | |
| // Transformer scratchpads | |
| cudaMalloc(&d_trans_h_, 8 * 256 * sizeof(float)); | |
| cudaMalloc(&d_trans_norm_, 8 * 256 * sizeof(float)); | |
| cudaMalloc(&d_trans_qkv_, 8 * 768 * sizeof(float)); | |
| cudaMalloc(&d_trans_q_head_, 8 * 8 * 32 * sizeof(float)); | |
| cudaMalloc(&d_trans_k_head_, 8 * 16 * 32 * sizeof(float)); | |
| cudaMalloc(&d_trans_v_head_, 8 * 16 * 32 * sizeof(float)); | |
| cudaMalloc(&d_trans_scores_, 8 * 8 * 16 * sizeof(float)); | |
| cudaMalloc(&d_trans_ctx_head_, 8 * 8 * 32 * sizeof(float)); | |
| cudaMalloc(&d_trans_context_, 8 * 256 * sizeof(float)); | |
| cudaMalloc(&d_trans_out_, 8 * 256 * sizeof(float)); | |
| cudaMalloc(&d_trans_mlp_, 8 * 1024 * sizeof(float)); | |
| cudaMalloc(&d_trans_film_, 512 * sizeof(float)); | |
| cudaMalloc(&d_temp_k_, 16 * 256 * sizeof(float)); | |
| cudaMalloc(&d_temp_v_, 16 * 256 * sizeof(float)); | |
| // Precomputed Visual KV Cache scratchpads (4 layers x 8 heads x 16 keys x 32 dims) | |
| cudaMalloc(&d_cached_k_, 4 * 8 * 16 * 32 * sizeof(float)); | |
| cudaMalloc(&d_cached_v_, 4 * 8 * 16 * 32 * sizeof(float)); | |
| cudaMalloc(&d_q_proj_, 8 * 256 * sizeof(float)); | |
| cudaMalloc(&d_action_min_, 7 * sizeof(float)); | |
| cudaMalloc(&d_action_max_, 7 * sizeof(float)); | |
| cudaMalloc(&d_text_emb_, 256 * sizeof(float)); | |
| cudaMalloc(&d_raw_agent_, 256 * 256 * 3); | |
| cudaMalloc(&d_raw_wrist_, 256 * 256 * 3); | |
| cudaMalloc(&d_final_actions_, 8 * 7 * sizeof(float)); | |
| } | |
| void ParvulusEngine::free_scratchpads() { | |
| cudaFree(d_obs_); | |
| cudaFree(d_z_vis_); | |
| cudaFree(d_z_sigma_); | |
| cudaFree(d_v_pred_); | |
| cudaFree(d_cond_); | |
| cudaFree(d_t_emb_); | |
| cudaFree(d_vis_scratch_a_); | |
| cudaFree(d_vis_scratch_b_); | |
| cudaFree(d_vis_scratch_c_); | |
| cudaFree(d_vis_scratch_d_); | |
| cudaFree(d_im2col_); | |
| cudaFree(d_film_ss_); | |
| cudaFree(d_pool_k_); | |
| cudaFree(d_pool_v_); | |
| cudaFree(d_pool_k_head_); | |
| cudaFree(d_pool_v_head_); | |
| cudaFree(d_pool_q_); | |
| cudaFree(d_pool_q_head_); | |
| cudaFree(d_pool_scores_); | |
| cudaFree(d_pool_ctx_head_); | |
| cudaFree(d_pool_context_); | |
| if (d_gn_stats_) cudaFree(d_gn_stats_); | |
| for (int s = 0; s < 4; ++s) { | |
| if (d_attn_pool_wk_fused_[s]) cudaFree(d_attn_pool_wk_fused_[s]); | |
| if (d_attn_pool_bk_fused_[s]) cudaFree(d_attn_pool_bk_fused_[s]); | |
| if (d_attn_pool_wv_fused_[s]) cudaFree(d_attn_pool_wv_fused_[s]); | |
| if (d_attn_pool_bv_fused_[s]) cudaFree(d_attn_pool_bv_fused_[s]); | |
| if (d_attn_pool_q_head_[s]) cudaFree(d_attn_pool_q_head_[s]); | |
| } | |
| cudaFree(d_trans_h_); | |
| cudaFree(d_trans_norm_); | |
| cudaFree(d_trans_qkv_); | |
| cudaFree(d_trans_q_head_); | |
| cudaFree(d_trans_k_head_); | |
| cudaFree(d_trans_v_head_); | |
| cudaFree(d_trans_scores_); | |
| cudaFree(d_trans_ctx_head_); | |
| cudaFree(d_trans_context_); | |
| cudaFree(d_trans_out_); | |
| cudaFree(d_trans_mlp_); | |
| cudaFree(d_trans_film_); | |
| cudaFree(d_temp_k_); | |
| cudaFree(d_temp_v_); | |
| cudaFree(d_cached_k_); | |
| cudaFree(d_cached_v_); | |
| cudaFree(d_q_proj_); | |
| cudaFree(d_raw_agent_); | |
| cudaFree(d_raw_wrist_); | |
| cudaFree(d_final_actions_); | |
| } | |
| bool ParvulusEngine::load_weights(const std::string& weights_path) { | |
| std::ifstream file(weights_path, std::ios::binary); | |
| if (!file.is_open()) { | |
| std::cerr << "[ParvulusEngine] Failed to open weights file: " << weights_path << std::endl; | |
| return false; | |
| } | |
| uint32_t magic, version, num_tensors, header_size; | |
| file.read(reinterpret_cast<char*>(&magic), 4); | |
| file.read(reinterpret_cast<char*>(&version), 4); | |
| file.read(reinterpret_cast<char*>(&num_tensors), 4); | |
| file.read(reinterpret_cast<char*>(&header_size), 4); | |
| if (magic != 0x54524B33 && magic != 0x50415256) { | |
| std::cerr << "[ParvulusEngine] Invalid magic number in weight binary!" << std::endl; | |
| return false; | |
| } | |
| tensors_.clear(); | |
| size_t data_section_size = 0; | |
| for (uint32_t i = 0; i < num_tensors; ++i) { | |
| uint16_t name_len; | |
| file.read(reinterpret_cast<char*>(&name_len), 2); | |
| std::string name(name_len, '\0'); | |
| file.read(&name[0], name_len); | |
| uint16_t num_dims; | |
| uint32_t dims[4]; | |
| uint64_t offset, size_bytes; | |
| file.read(reinterpret_cast<char*>(&num_dims), 2); | |
| file.read(reinterpret_cast<char*>(dims), 4 * 4); | |
| file.read(reinterpret_cast<char*>(&offset), 8); | |
| file.read(reinterpret_cast<char*>(&size_bytes), 8); | |
| TensorInfo info; | |
| info.name = name; | |
| for (int d = 0; d < num_dims; ++d) info.shape.push_back(dims[d]); | |
| info.offset = offset; | |
| info.size_bytes = size_bytes; | |
| tensors_[name] = info; | |
| data_section_size = std::max(data_section_size, (size_t)(offset + size_bytes)); | |
| } | |
| total_weight_bytes_ = data_section_size; | |
| std::vector<char> h_weights_data(data_section_size); | |
| file.read(h_weights_data.data(), data_section_size); | |
| cudaMalloc(&d_weights_buffer_, total_weight_bytes_); | |
| cudaMemcpy(d_weights_buffer_, h_weights_data.data(), total_weight_bytes_, cudaMemcpyHostToDevice); | |
| // Map device pointers | |
| for (auto& pair : tensors_) { | |
| pair.second.d_ptr = reinterpret_cast<float*>(reinterpret_cast<char*>(d_weights_buffer_) + pair.second.offset); | |
| } | |
| // Precompute / Fuse AttnPool weights and static query heads for all 4 stages | |
| int stage_in_ch[4] = {96, 192, 192, 384}; | |
| float alpha = 1.0f, beta = 0.0f; | |
| for (int s = 0; s < 4; ++s) { | |
| int in_c = stage_in_ch[s]; | |
| std::string pfx = "encoder.attn_poolers." + std::to_string(s) + "."; | |
| float* w_ip = get_tensor(pfx + "in_proj.weight"); | |
| float* b_ip = get_tensor(pfx + "in_proj.bias"); | |
| float* w_qkv = get_tensor(pfx + "cross_attn.in_proj_weight"); | |
| float* b_qkv = get_tensor(pfx + "cross_attn.in_proj_bias"); | |
| float* q_in = get_tensor(pfx + "queries"); | |
| float* w_q = w_qkv; | |
| float* b_q = b_qkv; | |
| float* w_k = w_qkv + 256 * 256; | |
| float* b_k = b_qkv + 256; | |
| float* w_v = w_qkv + 512 * 256; | |
| float* b_v = b_qkv + 512; | |
| // 1. Fused W_K: in_c x 256 col-major | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_k, 256, &beta, d_attn_pool_wk_fused_[s], in_c); | |
| // Fused b_K: W_K * b_ip + b_K | |
| cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_k, 256, b_ip, 1, &beta, d_attn_pool_bk_fused_[s], 1); | |
| add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bk_fused_[s], b_k, 256); | |
| // 2. Fused W_V: in_c x 256 col-major | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_v, 256, &beta, d_attn_pool_wv_fused_[s], in_c); | |
| // Fused b_V: W_V * b_ip + b_V | |
| cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_v, 256, b_ip, 1, &beta, d_attn_pool_bv_fused_[s], 1); | |
| add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bv_fused_[s], b_v, 256); | |
| // 3. Precompute Q_head: | |
| // q_in [4, 256] * W_Q^T + b_Q -> temp in d_pool_q_ | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_q, 256, q_in, 256, &beta, d_pool_q_, 256); | |
| add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream_>>>(d_pool_q_, b_q, 4, 256); | |
| transpose_interleaved_to_head_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream_>>>( | |
| d_pool_q_, d_attn_pool_q_head_[s], 4, 4, 64 | |
| ); | |
| } | |
| cudaStreamSynchronize(stream_); | |
| std::cout << "[ParvulusEngine] Successfully loaded " << num_tensors << " weight tensors (" | |
| << (total_weight_bytes_ / (1024.0 * 1024.0)) << " MB) into GPU VRAM." << std::endl; | |
| return true; | |
| } | |
| bool ParvulusEngine::load_task_embeddings(const std::string& embeddings_path) { | |
| std::ifstream file(embeddings_path, std::ios::binary); | |
| if (!file.is_open()) { | |
| std::cerr << "[ParvulusEngine] Failed to open task embeddings: " << embeddings_path << std::endl; | |
| return false; | |
| } | |
| uint32_t num_tasks; | |
| file.read(reinterpret_cast<char*>(&num_tasks), 4); | |
| task_embeddings_.clear(); | |
| for (uint32_t i = 0; i < num_tasks; ++i) { | |
| uint16_t str_len; | |
| file.read(reinterpret_cast<char*>(&str_len), 2); | |
| std::string task_str(str_len, '\0'); | |
| file.read(&task_str[0], str_len); | |
| std::vector<float> emb(256); | |
| file.read(reinterpret_cast<char*>(emb.data()), 256 * sizeof(float)); | |
| task_embeddings_[task_str] = emb; | |
| } | |
| std::cout << "[ParvulusEngine] Loaded " << num_tasks << " pre-cached task language embeddings." << std::endl; | |
| return true; | |
| } | |
| bool ParvulusEngine::load_action_scale(const std::string& scale_path) { | |
| std::ifstream file(scale_path, std::ios::binary); | |
| if (!file.is_open()) { | |
| std::cerr << "[ParvulusEngine] Failed to open action scale: " << scale_path << std::endl; | |
| return false; | |
| } | |
| file.read(reinterpret_cast<char*>(h_action_min_), 7 * sizeof(float)); | |
| file.read(reinterpret_cast<char*>(h_action_max_), 7 * sizeof(float)); | |
| cudaMemcpy(d_action_min_, h_action_min_, 7 * sizeof(float), cudaMemcpyHostToDevice); | |
| cudaMemcpy(d_action_max_, h_action_max_, 7 * sizeof(float), cudaMemcpyHostToDevice); | |
| return true; | |
| } | |
| float* ParvulusEngine::get_tensor(const std::string& name) { | |
| auto it = tensors_.find(name); | |
| if (it == tensors_.end()) { | |
| std::cerr << "[ParvulusEngine] Error: Tensor not found: " << name << std::endl; | |
| return nullptr; | |
| } | |
| return it->second.d_ptr; | |
| } | |
| // ------------------------------------------------------------- | |
| // Precompute Visual Key/Value Cache (Evaluated ONCE per chunk) | |
| // ------------------------------------------------------------- | |
| void ParvulusEngine::precompute_kv_cache() { | |
| float alpha = 1.0f, beta = 0.0f; | |
| for (int layer = 0; layer < 4; ++layer) { | |
| std::string prefix = "action_decoder.layers." + std::to_string(layer) + "."; | |
| float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight"); | |
| float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias"); | |
| if (!w_c_qkv) continue; | |
| // In PyTorch MultiheadAttention in_proj_weight: shape [768, 256] | |
| // K weight: rows 256..511 (offset 256 * 256), bias: rows 256..511 (offset 256) | |
| // V weight: rows 512..767 (offset 512 * 256), bias: rows 512..767 (offset 512) | |
| float* w_k = w_c_qkv + 256 * 256; | |
| float* b_k = b_c_qkv + 256; | |
| float* w_v = w_c_qkv + 512 * 256; | |
| float* b_v = b_c_qkv + 512; | |
| float* out_k = d_cached_k_ + layer * 8 * 16 * 32; | |
| float* out_v = d_cached_v_ + layer * 8 * 16 * 32; | |
| // 1. K = z_vis * W_K^T + b_K: [16, 256] x [256, 256]^T -> [16, 256] | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_k, 256, d_z_vis_, 256, &beta, d_temp_k_, 256); | |
| add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_k_, b_k, 16, 256); | |
| // 2. V = z_vis * W_V^T + b_V: [16, 256] x [256, 256]^T -> [16, 256] | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_v, 256, d_z_vis_, 256, &beta, d_temp_v_, 256); | |
| add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_v_, b_v, 16, 256); | |
| // 3. Transpose to head-major: [16 tokens, 8 heads, 32 dims] -> [8 heads, 16 tokens, 32 dims] | |
| transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( | |
| d_temp_k_, out_k, 16, 8, 32 | |
| ); | |
| transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( | |
| d_temp_v_, out_v, 16, 8, 32 | |
| ); | |
| } | |
| } | |
| // ------------------------------------------------------------- | |
| // Timestep Conditioning MLP | |
| // ------------------------------------------------------------- | |
| void ParvulusEngine::forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond) { | |
| float alpha = 1.0f, beta = 0.0f; | |
| sinusoidal_embedding_kernel<<<1, 128, 0, stream_>>>(t, d_t_emb_, 256); | |
| float* w0 = get_tensor("action_t_mlp.mlp.0.weight"); | |
| float* b0 = get_tensor("action_t_mlp.mlp.0.bias"); | |
| cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w0, 256, d_t_emb_, 1, &beta, d_trans_norm_, 1); | |
| add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, b0, 256); | |
| silu_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_norm_, 256); | |
| float* w2 = get_tensor("action_t_mlp.mlp.2.weight"); | |
| float* b2 = get_tensor("action_t_mlp.mlp.2.bias"); | |
| cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w2, 256, d_trans_norm_, 1, &beta, d_t_emb_, 1); | |
| add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_t_emb_, b2, 256); | |
| add_residual_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_text_emb, d_t_emb_, d_cond, 256); | |
| } | |
| // ------------------------------------------------------------- | |
| // Fast KV-Cached Transformer Decoder Step | |
| // ------------------------------------------------------------- | |
| void ParvulusEngine::forward_decoder_kv( | |
| const float* d_z_sigma_8x7, | |
| const float* d_cond_256, | |
| float* d_v_pred_8x7 | |
| ) { | |
| float alpha = 1.0f, beta = 0.0f; | |
| // 1. Initial linear projection + pos_emb: h = z_sigma * W_in^T + b_in + pos_emb | |
| float* w_in = get_tensor("action_decoder.action_in_proj.weight"); | |
| float* b_in = get_tensor("action_decoder.action_in_proj.bias"); | |
| float* w_pos = get_tensor("action_decoder.pos_emb"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 7, &alpha, w_in, 7, d_z_sigma_8x7, 7, &beta, d_trans_h_, 256); | |
| add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, b_in, 8, 256); | |
| add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, w_pos, d_trans_h_, 8 * 256); | |
| float scale = 0.1767766952966369f; // 1.0 / sqrt(32) | |
| // 2. Loop over 4 Transformer Decoder Layers | |
| for (int layer = 0; layer < 4; ++layer) { | |
| std::string prefix = "action_decoder.layers." + std::to_string(layer) + "."; | |
| // ------------------------------------------------------------- | |
| // (a) Temporal Self-Attention | |
| // ------------------------------------------------------------- | |
| float* g_n1 = get_tensor(prefix + "norm1.weight"); | |
| float* b_n1 = get_tensor(prefix + "norm1.bias"); | |
| layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n1, b_n1, d_trans_norm_, 8); | |
| float* w_s_qkv = get_tensor(prefix + "self_attn.in_proj_weight"); | |
| float* b_s_qkv = get_tensor(prefix + "self_attn.in_proj_bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 768, 8, 256, &alpha, w_s_qkv, 256, d_trans_norm_, 256, &beta, d_trans_qkv_, 768); | |
| add_bias_across_tokens_kernel<<<(8 * 768 + 255) / 256, 256, 0, stream_>>>(d_trans_qkv_, b_s_qkv, 8, 768); | |
| // Transpose QKV to head-major [8 heads, 8 tokens, 32 dims] | |
| transpose_qkv_to_heads_kernel<<<8, 256, 0, stream_>>>( | |
| d_trans_qkv_, d_trans_q_head_, d_trans_k_head_, d_trans_v_head_, 8, 8, 32 | |
| ); | |
| // Scores = Q_h * K_h^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 8 keys] | |
| cublasSgemmStridedBatched( | |
| cublas_handle_, | |
| CUBLAS_OP_T, CUBLAS_OP_N, | |
| 8, 8, 32, | |
| &scale, | |
| d_trans_k_head_, 32, 256, | |
| d_trans_q_head_, 32, 256, | |
| &beta, | |
| d_trans_scores_, 8, 64, | |
| 8 | |
| ); | |
| // Softmax across 8 keys | |
| softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 8); | |
| // Context = Scores * V_h -> [8 heads, 8 queries, 32 dims] | |
| cublasSgemmStridedBatched( | |
| cublas_handle_, | |
| CUBLAS_OP_N, CUBLAS_OP_N, | |
| 32, 8, 8, | |
| &alpha, | |
| d_trans_v_head_, 32, 256, | |
| d_trans_scores_, 8, 64, | |
| &beta, | |
| d_trans_ctx_head_, 32, 256, | |
| 8 | |
| ); | |
| // Transpose context to interleaved [8 tokens, 256 dims] | |
| transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( | |
| d_trans_ctx_head_, d_trans_context_, 8, 8, 32 | |
| ); | |
| // Out projection & residual | |
| float* w_s_out = get_tensor(prefix + "self_attn.out_proj.weight"); | |
| float* b_s_out = get_tensor(prefix + "self_attn.out_proj.bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_s_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256); | |
| add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_s_out, 8, 256); | |
| add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256); | |
| // ------------------------------------------------------------- | |
| // (b) Visual Cross-Attention with Cached K and V | |
| // ------------------------------------------------------------- | |
| float* g_n2 = get_tensor(prefix + "norm2.weight"); | |
| float* b_n2 = get_tensor(prefix + "norm2.bias"); | |
| layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n2, b_n2, d_trans_norm_, 8); | |
| float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight"); | |
| float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_qkv, 256, d_trans_norm_, 256, &beta, d_q_proj_, 256); | |
| add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_q_proj_, b_c_qkv, 8, 256); | |
| // Transpose Q to head-major [8 heads, 8 queries, 32 dims] | |
| transpose_interleaved_to_head_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( | |
| d_q_proj_, d_trans_q_head_, 8, 8, 32 | |
| ); | |
| float* cached_k = d_cached_k_ + layer * 8 * 16 * 32; | |
| float* cached_v = d_cached_v_ + layer * 8 * 16 * 32; | |
| // Scores = Q_h * K_cached^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 16 keys] | |
| cublasSgemmStridedBatched( | |
| cublas_handle_, | |
| CUBLAS_OP_T, CUBLAS_OP_N, | |
| 16, 8, 32, | |
| &scale, | |
| cached_k, 32, 512, | |
| d_trans_q_head_, 32, 256, | |
| &beta, | |
| d_trans_scores_, 16, 128, | |
| 8 | |
| ); | |
| // Softmax across 16 keys | |
| softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 16); | |
| // Context = Scores * V_cached -> [8 heads, 8 queries, 32 dims] | |
| cublasSgemmStridedBatched( | |
| cublas_handle_, | |
| CUBLAS_OP_N, CUBLAS_OP_N, | |
| 32, 8, 16, | |
| &alpha, | |
| cached_v, 32, 512, | |
| d_trans_scores_, 16, 128, | |
| &beta, | |
| d_trans_ctx_head_, 32, 256, | |
| 8 | |
| ); | |
| // Transpose context to interleaved [8 tokens, 256 dims] | |
| transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>( | |
| d_trans_ctx_head_, d_trans_context_, 8, 8, 32 | |
| ); | |
| // Out projection & residual | |
| float* w_c_out = get_tensor(prefix + "cross_attn.out_proj.weight"); | |
| float* b_c_out = get_tensor(prefix + "cross_attn.out_proj.bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256); | |
| add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_c_out, 8, 256); | |
| add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256); | |
| // ------------------------------------------------------------- | |
| // (c) FiLM Modulation & MLP | |
| // ------------------------------------------------------------- | |
| float* w_film = get_tensor(prefix + "film.weight"); | |
| float* b_film = get_tensor(prefix + "film.bias"); | |
| cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 512, &alpha, w_film, 256, d_cond_256, 1, &beta, d_trans_film_, 1); | |
| add_bias_1d_kernel<<<(512 + 255) / 256, 256, 0, stream_>>>(d_trans_film_, b_film, 512); | |
| float* g_n3 = get_tensor(prefix + "norm3.weight"); | |
| float* b_n3 = get_tensor(prefix + "norm3.bias"); | |
| layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n3, b_n3, d_trans_norm_, 8); | |
| film_modulate_256_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_film_, d_trans_norm_, 8); | |
| float* w_m1 = get_tensor(prefix + "mlp.0.weight"); | |
| float* b_m1 = get_tensor(prefix + "mlp.0.bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 1024, 8, 256, &alpha, w_m1, 256, d_trans_norm_, 256, &beta, d_trans_mlp_, 1024); | |
| add_bias_across_tokens_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, b_m1, 8, 1024); | |
| gelu_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, d_trans_mlp_, 8 * 1024); | |
| float* w_m2 = get_tensor(prefix + "mlp.2.weight"); | |
| float* b_m2 = get_tensor(prefix + "mlp.2.bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 1024, &alpha, w_m2, 1024, d_trans_mlp_, 1024, &beta, d_trans_out_, 256); | |
| add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_m2, 8, 256); | |
| add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256); | |
| } | |
| // 3. Final LayerNorm & Out Head (256 -> 7) | |
| float* g_fn = get_tensor("action_decoder.final_norm.weight"); | |
| float* b_fn = get_tensor("action_decoder.final_norm.bias"); | |
| layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_fn, b_fn, d_trans_norm_, 8); | |
| float* w_out = get_tensor("action_decoder.out_head.weight"); | |
| float* b_out = get_tensor("action_decoder.out_head.bias"); | |
| cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 7, 8, 256, &alpha, w_out, 256, d_trans_norm_, 256, &beta, d_v_pred_8x7, 7); | |
| add_bias_across_tokens_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_v_pred_8x7, b_out, 8, 7); | |
| } | |
| // ------------------------------------------------------------- | |
| // Vision DualViewEncoder Forward Pass | |
| // ------------------------------------------------------------- | |
| void ParvulusEngine::forward_encoder( | |
| const float* d_obs_6x256x256, | |
| const float* d_text_emb_256, | |
| float* d_z_vis_16x256 | |
| ) { | |
| dual_view_encoder_forward( | |
| this, | |
| cublas_handle_, | |
| stream_, | |
| d_obs_6x256x256, | |
| d_text_emb_256, | |
| d_z_vis_16x256, | |
| d_vis_scratch_a_, | |
| d_vis_scratch_b_, | |
| d_vis_scratch_c_, | |
| d_vis_scratch_d_, | |
| d_im2col_, | |
| d_film_ss_, | |
| d_pool_k_, | |
| d_pool_v_, | |
| d_pool_k_head_, | |
| d_pool_v_head_, | |
| d_pool_q_, | |
| d_pool_q_head_, | |
| d_pool_scores_, | |
| d_pool_ctx_head_, | |
| d_pool_context_ | |
| ); | |
| } | |
| // ------------------------------------------------------------- | |
| bool ParvulusEngine::register_task_embedding(const std::string& task_description, const float* emb_256) { | |
| if (!emb_256) return false; | |
| task_embeddings_[task_description] = std::vector<float>(emb_256, emb_256 + 256); | |
| return true; | |
| } | |
| // ------------------------------------------------------------- | |
| // Complete Euler Action Flow Prediction Pipeline (Vector-First) | |
| // ------------------------------------------------------------- | |
| bool ParvulusEngine::set_task_vector(const float* h_task_vector_256) { | |
| if (!h_task_vector_256) return false; | |
| cudaMemcpyAsync(d_text_emb_, h_task_vector_256, 256 * sizeof(float), cudaMemcpyHostToDevice, stream_); | |
| has_task_vector_ = true; | |
| return true; | |
| } | |
| bool ParvulusEngine::set_task_string(const std::string& task_description) { | |
| auto it = task_embeddings_.find(task_description); | |
| if (it == task_embeddings_.end()) { | |
| return false; | |
| } | |
| return set_task_vector(it->second.data()); | |
| } | |
| bool ParvulusEngine::predict_action_chunk( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| float* h_out_actions, | |
| int num_euler_steps, | |
| const float* custom_z0 | |
| ) { | |
| if (!has_task_vector_) { | |
| std::cerr << "[ParvulusEngine] Error: Task conditioning vector has not been set! Call set_task_vector() first." << std::endl; | |
| return false; | |
| } | |
| if (!h_agent_img || !h_wrist_img || !h_out_actions) { | |
| return false; | |
| } | |
| // 1. Preprocess raw RGB frames into normalized [6, 256, 256] tensor | |
| cudaMemcpyAsync(d_raw_agent_, h_agent_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_); | |
| cudaMemcpyAsync(d_raw_wrist_, h_wrist_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_); | |
| preprocess_frames_kernel<<<(256 * 256 + 255) / 256, 256, 0, stream_>>>( | |
| d_raw_agent_, d_raw_wrist_, d_obs_, 256, 256 | |
| ); | |
| // 2. Vision Forward Pass (Evaluated ONCE per chunk) | |
| forward_encoder(d_obs_, d_text_emb_, d_z_vis_); | |
| // 3. Precompute Visual KV Cache (Evaluated ONCE per chunk) | |
| precompute_kv_cache(); | |
| // 4. Sample base noise z0 | |
| if (custom_z0 != nullptr) { | |
| cudaMemcpyAsync(d_z_sigma_, custom_z0, 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_); | |
| } else { | |
| std::vector<float> h_z0(8 * 7); | |
| for (int i = 0; i < 8 * 7; ++i) h_z0[i] = norm_dist_(rng_); | |
| cudaMemcpyAsync(d_z_sigma_, h_z0.data(), 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_); | |
| } | |
| // 5. Euler ODE Integration Loop with KV Cache | |
| float dt = 1.0f / (float)num_euler_steps; | |
| for (int step = 0; step < num_euler_steps; ++step) { | |
| float t = (float)step * dt; | |
| forward_timestep_mlp(t, d_text_emb_, d_cond_); | |
| forward_decoder_kv(d_z_sigma_, d_cond_, d_v_pred_); | |
| euler_step_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, d_v_pred_, dt, 8 * 7); | |
| } | |
| clamp_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, -1.0f, 1.0f, 8 * 7); | |
| // 6. Denormalize actions to unnormalized robot space | |
| denormalize_actions_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>( | |
| d_z_sigma_, d_action_min_, d_action_max_, d_final_actions_, 8, 7 | |
| ); | |
| cudaMemcpyAsync(h_out_actions, d_final_actions_, 8 * 7 * sizeof(float), cudaMemcpyDeviceToHost, stream_); | |
| cudaStreamSynchronize(stream_); | |
| return true; | |
| } | |
| bool ParvulusEngine::predict_action_chunk_with_vector( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| const float* h_task_vector_256, | |
| float* h_out_actions, | |
| int num_euler_steps, | |
| const float* custom_z0 | |
| ) { | |
| if (!set_task_vector(h_task_vector_256)) return false; | |
| return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0); | |
| } | |
| bool ParvulusEngine::predict_action_chunk( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| const std::string& task_description, | |
| float* h_out_actions, | |
| int num_euler_steps, | |
| const float* custom_z0 | |
| ) { | |
| if (!set_task_string(task_description)) return false; | |
| return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0); | |
| } | |