Download c_engine/parvulus_engine.h from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 7.77 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/parvulus_engine.h
- Command line
-
hf download hf://aasivas/Parvulus/c_engine/parvulus_engine.h
-
curl -L -o parvulus_engine.h https://huggingface.co/aasivas/Parvulus/resolve/main/c_engine/parvulus_engine.h
7.77 kB
| struct TensorInfo { | |
| std::string name; | |
| std::vector<int> shape; | |
| size_t offset; | |
| size_t size_bytes; | |
| float* d_ptr = nullptr; // Device memory pointer | |
| }; | |
| class ParvulusEngine { | |
| public: | |
| ParvulusEngine(int device_id = 0); | |
| ~ParvulusEngine(); | |
| bool load_weights(const std::string& weights_path); | |
| bool load_task_embeddings(const std::string& embeddings_path); | |
| bool load_action_scale(const std::string& scale_path); | |
| bool register_task_embedding(const std::string& task_description, const float* emb_256); | |
| // Sets the active task conditioning vector on GPU (256 floats). | |
| // All subsequent chunk predictions will use this conditioning vector. | |
| bool set_task_vector(const float* h_task_vector_256); | |
| // Looks up a task string from cached dictionary and sets active conditioning vector. | |
| bool set_task_string(const std::string& task_description); | |
| // Predicts 8x7 action chunk using currently active task conditioning vector. | |
| // agent_img, wrist_img: [256 * 256 * 3] uint8 in HWC format | |
| // out_actions: float buffer of size [8 * 7] (56 floats) | |
| bool predict_action_chunk( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| float* h_out_actions, | |
| int num_euler_steps = 4, | |
| const float* custom_z0 = nullptr | |
| ); | |
| // Predicts 8x7 action chunk directly using a 256-dim task conditioning vector | |
| bool predict_action_chunk_with_vector( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| const float* h_task_vector_256, | |
| float* h_out_actions, | |
| int num_euler_steps = 4, | |
| const float* custom_z0 = nullptr | |
| ); | |
| // Alias for predict_action_chunk_with_vector | |
| bool predict_action_chunk_with_emb( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| const float* h_text_emb, | |
| float* h_out_actions, | |
| int num_euler_steps = 4, | |
| const float* custom_z0 = nullptr | |
| ) { | |
| return predict_action_chunk_with_vector(h_agent_img, h_wrist_img, h_text_emb, h_out_actions, num_euler_steps, custom_z0); | |
| } | |
| // Predicts 8x7 action chunk using a task string lookup | |
| bool predict_action_chunk( | |
| const uint8_t* h_agent_img, | |
| const uint8_t* h_wrist_img, | |
| const std::string& task_description, | |
| float* h_out_actions, | |
| int num_euler_steps = 4, | |
| const float* custom_z0 = nullptr | |
| ); | |
| // Direct tensor-to-tensor forward pass for parity verification | |
| void forward_encoder(const float* d_obs_6x256x256, const float* d_text_emb_256, float* d_z_vis_16x256); | |
| void forward_decoder(const float* d_z_sigma_8x7, const float* d_z_vis_16x256, const float* d_cond_256, float* d_v_pred_8x7); | |
| void set_seed(unsigned int seed) { rng_.seed(seed); } | |
| float* get_tensor(const std::string& name); | |
| // Optimized Vision Encoder Buffers & Precomputed Weights | |
| cudaStream_t get_stream() const { return stream_; } | |
| float* get_gn_stats() const { return d_gn_stats_; } | |
| float* get_attn_wk_fused(int s) const { return d_attn_pool_wk_fused_[s]; } | |
| float* get_attn_bk_fused(int s) const { return d_attn_pool_bk_fused_[s]; } | |
| float* get_attn_wv_fused(int s) const { return d_attn_pool_wv_fused_[s]; } | |
| float* get_attn_bv_fused(int s) const { return d_attn_pool_bv_fused_[s]; } | |
| float* get_attn_q_head(int s) const { return d_attn_pool_q_head_[s]; } | |
| private: | |
| int device_id_; | |
| cublasHandle_t cublas_handle_; | |
| cudaStream_t stream_; | |
| std::mt19937 rng_; | |
| std::normal_distribution<float> norm_dist_{0.0f, 1.0f}; | |
| // Weight storage | |
| float* d_weights_buffer_ = nullptr; | |
| size_t total_weight_bytes_ = 0; | |
| std::unordered_map<std::string, TensorInfo> tensors_; | |
| // Task text embeddings lookup | |
| std::unordered_map<std::string, std::vector<float>> task_embeddings_; | |
| float* d_text_emb_ = nullptr; | |
| bool has_task_vector_ = false; | |
| // Action scale bounds | |
| float h_action_min_[7]; | |
| float h_action_max_[7]; | |
| float* d_action_min_ = nullptr; | |
| float* d_action_max_ = nullptr; | |
| // Static scratchpad device memory buffers (Zero allocations at runtime) | |
| uint8_t* d_raw_agent_ = nullptr; // [256, 256, 3] | |
| uint8_t* d_raw_wrist_ = nullptr; // [256, 256, 3] | |
| float* d_final_actions_ = nullptr; // [8, 7] | |
| float* d_obs_ = nullptr; // [6, 256, 256] | |
| float* d_z_vis_ = nullptr; // [16, 256] | |
| float* d_z_sigma_ = nullptr; // [8, 7] | |
| float* d_v_pred_ = nullptr; // [8, 7] | |
| float* d_cond_ = nullptr; // [256] | |
| float* d_t_emb_ = nullptr; // [256] | |
| // Intermediate vision activations scratchpad | |
| float* d_vis_scratch_a_ = nullptr; // Large scratchpad A | |
| float* d_vis_scratch_b_ = nullptr; // Large scratchpad B | |
| float* d_vis_scratch_c_ = nullptr; // Large scratchpad C | |
| float* d_vis_scratch_d_ = nullptr; // Large scratchpad D | |
| float* d_im2col_ = nullptr; // Convolution im2col scratchpad | |
| float* d_film_ss_ = nullptr; // FiLM scale/shift scratchpad | |
| float* d_pool_k_ = nullptr; // AttnPool K scratchpad | |
| float* d_pool_v_ = nullptr; // AttnPool V scratchpad | |
| float* d_pool_k_head_ = nullptr; // AttnPool K head scratchpad | |
| float* d_pool_v_head_ = nullptr; // AttnPool V head scratchpad | |
| float* d_pool_q_ = nullptr; // AttnPool Q scratchpad | |
| float* d_pool_q_head_ = nullptr; // AttnPool Q head scratchpad | |
| float* d_pool_scores_ = nullptr; // AttnPool scores scratchpad | |
| float* d_pool_ctx_head_ = nullptr; // AttnPool context head scratchpad | |
| float* d_pool_context_ = nullptr; // AttnPool context scratchpad | |
| float* d_gn_stats_ = nullptr; // GroupNorm multi-block atomic reduction scratchpad (16 floats) | |
| // Precomputed / Fused AttnPool weights for stages 0..3 (96, 192, 192, 384 channels) | |
| float* d_attn_pool_wk_fused_[4] = {nullptr}; | |
| float* d_attn_pool_bk_fused_[4] = {nullptr}; | |
| float* d_attn_pool_wv_fused_[4] = {nullptr}; | |
| float* d_attn_pool_bv_fused_[4] = {nullptr}; | |
| float* d_attn_pool_q_head_[4] = {nullptr}; | |
| // Transformer scratchpad | |
| float* d_trans_h_ = nullptr; // [8, 256] | |
| float* d_trans_norm_ = nullptr; // [8, 256] | |
| float* d_trans_qkv_ = nullptr; // [8, 768] | |
| float* d_trans_q_head_ = nullptr; // [8, 8, 32] | |
| float* d_trans_k_head_ = nullptr; // [8, 16, 32] | |
| float* d_trans_v_head_ = nullptr; // [8, 16, 32] | |
| float* d_trans_scores_ = nullptr; // [8, 8, 16] | |
| float* d_trans_ctx_head_ = nullptr; // [8, 8, 32] | |
| float* d_trans_context_ = nullptr; // [8, 256] | |
| float* d_trans_out_ = nullptr; // [8, 256] | |
| float* d_trans_mlp_ = nullptr; // [8, 1024] | |
| float* d_trans_film_ = nullptr; // [512] | |
| float* d_temp_k_ = nullptr; // [16, 256] | |
| float* d_temp_v_ = nullptr; // [16, 256] | |
| // Precomputed Visual KV Cache buffers | |
| float* d_cached_k_ = nullptr; // [4, 8, 16, 32] | |
| float* d_cached_v_ = nullptr; // [4, 8, 16, 32] | |
| float* d_q_proj_ = nullptr; // [8, 256] | |
| // Native CUDA Graph capture for ODE loop | |
| cudaGraph_t ode_graph_; | |
| cudaGraphExec_t ode_graph_exec_ = nullptr; | |
| bool graph_captured_ = false; | |
| friend class DiagnosticRunner; | |
| friend class Profiler; | |
| void precompute_kv_cache(); | |
| void forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond); | |
| void forward_decoder_kv(const float* d_z_sigma_8x7, const float* d_cond_256, float* d_v_pred_8x7); | |
| void allocate_scratchpads(); | |
| void free_scratchpads(); | |
| }; | |