File size: 7,773 Bytes
67c4f10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 | #pragma once
#include <cuda_runtime.h>
#include <cublas_v2.h>
#include <string>
#include <vector>
#include <unordered_map>
#include <cstdint>
#include <random>
struct TensorInfo {
std::string name;
std::vector<int> shape;
size_t offset;
size_t size_bytes;
float* d_ptr = nullptr; // Device memory pointer
};
class ParvulusEngine {
public:
ParvulusEngine(int device_id = 0);
~ParvulusEngine();
bool load_weights(const std::string& weights_path);
bool load_task_embeddings(const std::string& embeddings_path);
bool load_action_scale(const std::string& scale_path);
bool register_task_embedding(const std::string& task_description, const float* emb_256);
// Sets the active task conditioning vector on GPU (256 floats).
// All subsequent chunk predictions will use this conditioning vector.
bool set_task_vector(const float* h_task_vector_256);
// Looks up a task string from cached dictionary and sets active conditioning vector.
bool set_task_string(const std::string& task_description);
// Predicts 8x7 action chunk using currently active task conditioning vector.
// agent_img, wrist_img: [256 * 256 * 3] uint8 in HWC format
// out_actions: float buffer of size [8 * 7] (56 floats)
bool predict_action_chunk(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
float* h_out_actions,
int num_euler_steps = 4,
const float* custom_z0 = nullptr
);
// Predicts 8x7 action chunk directly using a 256-dim task conditioning vector
bool predict_action_chunk_with_vector(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
const float* h_task_vector_256,
float* h_out_actions,
int num_euler_steps = 4,
const float* custom_z0 = nullptr
);
// Alias for predict_action_chunk_with_vector
bool predict_action_chunk_with_emb(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
const float* h_text_emb,
float* h_out_actions,
int num_euler_steps = 4,
const float* custom_z0 = nullptr
) {
return predict_action_chunk_with_vector(h_agent_img, h_wrist_img, h_text_emb, h_out_actions, num_euler_steps, custom_z0);
}
// Predicts 8x7 action chunk using a task string lookup
bool predict_action_chunk(
const uint8_t* h_agent_img,
const uint8_t* h_wrist_img,
const std::string& task_description,
float* h_out_actions,
int num_euler_steps = 4,
const float* custom_z0 = nullptr
);
// Direct tensor-to-tensor forward pass for parity verification
void forward_encoder(const float* d_obs_6x256x256, const float* d_text_emb_256, float* d_z_vis_16x256);
void forward_decoder(const float* d_z_sigma_8x7, const float* d_z_vis_16x256, const float* d_cond_256, float* d_v_pred_8x7);
void set_seed(unsigned int seed) { rng_.seed(seed); }
float* get_tensor(const std::string& name);
// Optimized Vision Encoder Buffers & Precomputed Weights
cudaStream_t get_stream() const { return stream_; }
float* get_gn_stats() const { return d_gn_stats_; }
float* get_attn_wk_fused(int s) const { return d_attn_pool_wk_fused_[s]; }
float* get_attn_bk_fused(int s) const { return d_attn_pool_bk_fused_[s]; }
float* get_attn_wv_fused(int s) const { return d_attn_pool_wv_fused_[s]; }
float* get_attn_bv_fused(int s) const { return d_attn_pool_bv_fused_[s]; }
float* get_attn_q_head(int s) const { return d_attn_pool_q_head_[s]; }
private:
int device_id_;
cublasHandle_t cublas_handle_;
cudaStream_t stream_;
std::mt19937 rng_;
std::normal_distribution<float> norm_dist_{0.0f, 1.0f};
// Weight storage
float* d_weights_buffer_ = nullptr;
size_t total_weight_bytes_ = 0;
std::unordered_map<std::string, TensorInfo> tensors_;
// Task text embeddings lookup
std::unordered_map<std::string, std::vector<float>> task_embeddings_;
float* d_text_emb_ = nullptr;
bool has_task_vector_ = false;
// Action scale bounds
float h_action_min_[7];
float h_action_max_[7];
float* d_action_min_ = nullptr;
float* d_action_max_ = nullptr;
// Static scratchpad device memory buffers (Zero allocations at runtime)
uint8_t* d_raw_agent_ = nullptr; // [256, 256, 3]
uint8_t* d_raw_wrist_ = nullptr; // [256, 256, 3]
float* d_final_actions_ = nullptr; // [8, 7]
float* d_obs_ = nullptr; // [6, 256, 256]
float* d_z_vis_ = nullptr; // [16, 256]
float* d_z_sigma_ = nullptr; // [8, 7]
float* d_v_pred_ = nullptr; // [8, 7]
float* d_cond_ = nullptr; // [256]
float* d_t_emb_ = nullptr; // [256]
// Intermediate vision activations scratchpad
float* d_vis_scratch_a_ = nullptr; // Large scratchpad A
float* d_vis_scratch_b_ = nullptr; // Large scratchpad B
float* d_vis_scratch_c_ = nullptr; // Large scratchpad C
float* d_vis_scratch_d_ = nullptr; // Large scratchpad D
float* d_im2col_ = nullptr; // Convolution im2col scratchpad
float* d_film_ss_ = nullptr; // FiLM scale/shift scratchpad
float* d_pool_k_ = nullptr; // AttnPool K scratchpad
float* d_pool_v_ = nullptr; // AttnPool V scratchpad
float* d_pool_k_head_ = nullptr; // AttnPool K head scratchpad
float* d_pool_v_head_ = nullptr; // AttnPool V head scratchpad
float* d_pool_q_ = nullptr; // AttnPool Q scratchpad
float* d_pool_q_head_ = nullptr; // AttnPool Q head scratchpad
float* d_pool_scores_ = nullptr; // AttnPool scores scratchpad
float* d_pool_ctx_head_ = nullptr; // AttnPool context head scratchpad
float* d_pool_context_ = nullptr; // AttnPool context scratchpad
float* d_gn_stats_ = nullptr; // GroupNorm multi-block atomic reduction scratchpad (16 floats)
// Precomputed / Fused AttnPool weights for stages 0..3 (96, 192, 192, 384 channels)
float* d_attn_pool_wk_fused_[4] = {nullptr};
float* d_attn_pool_bk_fused_[4] = {nullptr};
float* d_attn_pool_wv_fused_[4] = {nullptr};
float* d_attn_pool_bv_fused_[4] = {nullptr};
float* d_attn_pool_q_head_[4] = {nullptr};
// Transformer scratchpad
float* d_trans_h_ = nullptr; // [8, 256]
float* d_trans_norm_ = nullptr; // [8, 256]
float* d_trans_qkv_ = nullptr; // [8, 768]
float* d_trans_q_head_ = nullptr; // [8, 8, 32]
float* d_trans_k_head_ = nullptr; // [8, 16, 32]
float* d_trans_v_head_ = nullptr; // [8, 16, 32]
float* d_trans_scores_ = nullptr; // [8, 8, 16]
float* d_trans_ctx_head_ = nullptr; // [8, 8, 32]
float* d_trans_context_ = nullptr; // [8, 256]
float* d_trans_out_ = nullptr; // [8, 256]
float* d_trans_mlp_ = nullptr; // [8, 1024]
float* d_trans_film_ = nullptr; // [512]
float* d_temp_k_ = nullptr; // [16, 256]
float* d_temp_v_ = nullptr; // [16, 256]
// Precomputed Visual KV Cache buffers
float* d_cached_k_ = nullptr; // [4, 8, 16, 32]
float* d_cached_v_ = nullptr; // [4, 8, 16, 32]
float* d_q_proj_ = nullptr; // [8, 256]
// Native CUDA Graph capture for ODE loop
cudaGraph_t ode_graph_;
cudaGraphExec_t ode_graph_exec_ = nullptr;
bool graph_captured_ = false;
friend class DiagnosticRunner;
friend class Profiler;
void precompute_kv_cache();
void forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond);
void forward_decoder_kv(const float* d_z_sigma_8x7, const float* d_cond_256, float* d_v_pred_8x7);
void allocate_scratchpads();
void free_scratchpads();
};
|