#pragma once #include #include #include #include #include #include #include struct TensorInfo { std::string name; std::vector shape; size_t offset; size_t size_bytes; float* d_ptr = nullptr; // Device memory pointer }; class ParvulusEngine { public: ParvulusEngine(int device_id = 0); ~ParvulusEngine(); bool load_weights(const std::string& weights_path); bool load_task_embeddings(const std::string& embeddings_path); bool load_action_scale(const std::string& scale_path); bool register_task_embedding(const std::string& task_description, const float* emb_256); // Sets the active task conditioning vector on GPU (256 floats). // All subsequent chunk predictions will use this conditioning vector. bool set_task_vector(const float* h_task_vector_256); // Looks up a task string from cached dictionary and sets active conditioning vector. bool set_task_string(const std::string& task_description); // Predicts 8x7 action chunk using currently active task conditioning vector. // agent_img, wrist_img: [256 * 256 * 3] uint8 in HWC format // out_actions: float buffer of size [8 * 7] (56 floats) bool predict_action_chunk( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, float* h_out_actions, int num_euler_steps = 4, const float* custom_z0 = nullptr ); // Predicts 8x7 action chunk directly using a 256-dim task conditioning vector bool predict_action_chunk_with_vector( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, const float* h_task_vector_256, float* h_out_actions, int num_euler_steps = 4, const float* custom_z0 = nullptr ); // Alias for predict_action_chunk_with_vector bool predict_action_chunk_with_emb( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, const float* h_text_emb, float* h_out_actions, int num_euler_steps = 4, const float* custom_z0 = nullptr ) { return predict_action_chunk_with_vector(h_agent_img, h_wrist_img, h_text_emb, h_out_actions, num_euler_steps, custom_z0); } // Predicts 8x7 action chunk using a task string lookup bool predict_action_chunk( const uint8_t* h_agent_img, const uint8_t* h_wrist_img, const std::string& task_description, float* h_out_actions, int num_euler_steps = 4, const float* custom_z0 = nullptr ); // Direct tensor-to-tensor forward pass for parity verification void forward_encoder(const float* d_obs_6x256x256, const float* d_text_emb_256, float* d_z_vis_16x256); void forward_decoder(const float* d_z_sigma_8x7, const float* d_z_vis_16x256, const float* d_cond_256, float* d_v_pred_8x7); void set_seed(unsigned int seed) { rng_.seed(seed); } float* get_tensor(const std::string& name); // Optimized Vision Encoder Buffers & Precomputed Weights cudaStream_t get_stream() const { return stream_; } float* get_gn_stats() const { return d_gn_stats_; } float* get_attn_wk_fused(int s) const { return d_attn_pool_wk_fused_[s]; } float* get_attn_bk_fused(int s) const { return d_attn_pool_bk_fused_[s]; } float* get_attn_wv_fused(int s) const { return d_attn_pool_wv_fused_[s]; } float* get_attn_bv_fused(int s) const { return d_attn_pool_bv_fused_[s]; } float* get_attn_q_head(int s) const { return d_attn_pool_q_head_[s]; } private: int device_id_; cublasHandle_t cublas_handle_; cudaStream_t stream_; std::mt19937 rng_; std::normal_distribution norm_dist_{0.0f, 1.0f}; // Weight storage float* d_weights_buffer_ = nullptr; size_t total_weight_bytes_ = 0; std::unordered_map tensors_; // Task text embeddings lookup std::unordered_map> task_embeddings_; float* d_text_emb_ = nullptr; bool has_task_vector_ = false; // Action scale bounds float h_action_min_[7]; float h_action_max_[7]; float* d_action_min_ = nullptr; float* d_action_max_ = nullptr; // Static scratchpad device memory buffers (Zero allocations at runtime) uint8_t* d_raw_agent_ = nullptr; // [256, 256, 3] uint8_t* d_raw_wrist_ = nullptr; // [256, 256, 3] float* d_final_actions_ = nullptr; // [8, 7] float* d_obs_ = nullptr; // [6, 256, 256] float* d_z_vis_ = nullptr; // [16, 256] float* d_z_sigma_ = nullptr; // [8, 7] float* d_v_pred_ = nullptr; // [8, 7] float* d_cond_ = nullptr; // [256] float* d_t_emb_ = nullptr; // [256] // Intermediate vision activations scratchpad float* d_vis_scratch_a_ = nullptr; // Large scratchpad A float* d_vis_scratch_b_ = nullptr; // Large scratchpad B float* d_vis_scratch_c_ = nullptr; // Large scratchpad C float* d_vis_scratch_d_ = nullptr; // Large scratchpad D float* d_im2col_ = nullptr; // Convolution im2col scratchpad float* d_film_ss_ = nullptr; // FiLM scale/shift scratchpad float* d_pool_k_ = nullptr; // AttnPool K scratchpad float* d_pool_v_ = nullptr; // AttnPool V scratchpad float* d_pool_k_head_ = nullptr; // AttnPool K head scratchpad float* d_pool_v_head_ = nullptr; // AttnPool V head scratchpad float* d_pool_q_ = nullptr; // AttnPool Q scratchpad float* d_pool_q_head_ = nullptr; // AttnPool Q head scratchpad float* d_pool_scores_ = nullptr; // AttnPool scores scratchpad float* d_pool_ctx_head_ = nullptr; // AttnPool context head scratchpad float* d_pool_context_ = nullptr; // AttnPool context scratchpad float* d_gn_stats_ = nullptr; // GroupNorm multi-block atomic reduction scratchpad (16 floats) // Precomputed / Fused AttnPool weights for stages 0..3 (96, 192, 192, 384 channels) float* d_attn_pool_wk_fused_[4] = {nullptr}; float* d_attn_pool_bk_fused_[4] = {nullptr}; float* d_attn_pool_wv_fused_[4] = {nullptr}; float* d_attn_pool_bv_fused_[4] = {nullptr}; float* d_attn_pool_q_head_[4] = {nullptr}; // Transformer scratchpad float* d_trans_h_ = nullptr; // [8, 256] float* d_trans_norm_ = nullptr; // [8, 256] float* d_trans_qkv_ = nullptr; // [8, 768] float* d_trans_q_head_ = nullptr; // [8, 8, 32] float* d_trans_k_head_ = nullptr; // [8, 16, 32] float* d_trans_v_head_ = nullptr; // [8, 16, 32] float* d_trans_scores_ = nullptr; // [8, 8, 16] float* d_trans_ctx_head_ = nullptr; // [8, 8, 32] float* d_trans_context_ = nullptr; // [8, 256] float* d_trans_out_ = nullptr; // [8, 256] float* d_trans_mlp_ = nullptr; // [8, 1024] float* d_trans_film_ = nullptr; // [512] float* d_temp_k_ = nullptr; // [16, 256] float* d_temp_v_ = nullptr; // [16, 256] // Precomputed Visual KV Cache buffers float* d_cached_k_ = nullptr; // [4, 8, 16, 32] float* d_cached_v_ = nullptr; // [4, 8, 16, 32] float* d_q_proj_ = nullptr; // [8, 256] // Native CUDA Graph capture for ODE loop cudaGraph_t ode_graph_; cudaGraphExec_t ode_graph_exec_ = nullptr; bool graph_captured_ = false; friend class DiagnosticRunner; friend class Profiler; void precompute_kv_cache(); void forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond); void forward_decoder_kv(const float* d_z_sigma_8x7, const float* d_cond_256, float* d_v_pred_8x7); void allocate_scratchpads(); void free_scratchpads(); };