File size: 7,773 Bytes
67c4f10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
#pragma once

#include <cuda_runtime.h>
#include <cublas_v2.h>
#include <string>
#include <vector>
#include <unordered_map>
#include <cstdint>
#include <random>

struct TensorInfo {
    std::string name;
    std::vector<int> shape;
    size_t offset;
    size_t size_bytes;
    float* d_ptr = nullptr;  // Device memory pointer
};

class ParvulusEngine {
public:
    ParvulusEngine(int device_id = 0);
    ~ParvulusEngine();

    bool load_weights(const std::string& weights_path);
    bool load_task_embeddings(const std::string& embeddings_path);
    bool load_action_scale(const std::string& scale_path);
    bool register_task_embedding(const std::string& task_description, const float* emb_256);

    // Sets the active task conditioning vector on GPU (256 floats).
    // All subsequent chunk predictions will use this conditioning vector.
    bool set_task_vector(const float* h_task_vector_256);

    // Looks up a task string from cached dictionary and sets active conditioning vector.
    bool set_task_string(const std::string& task_description);

    // Predicts 8x7 action chunk using currently active task conditioning vector.
    // agent_img, wrist_img: [256 * 256 * 3] uint8 in HWC format
    // out_actions: float buffer of size [8 * 7] (56 floats)
    bool predict_action_chunk(
        const uint8_t* h_agent_img,
        const uint8_t* h_wrist_img,
        float* h_out_actions,
        int num_euler_steps = 4,
        const float* custom_z0 = nullptr
    );

    // Predicts 8x7 action chunk directly using a 256-dim task conditioning vector
    bool predict_action_chunk_with_vector(
        const uint8_t* h_agent_img,
        const uint8_t* h_wrist_img,
        const float* h_task_vector_256,
        float* h_out_actions,
        int num_euler_steps = 4,
        const float* custom_z0 = nullptr
    );

    // Alias for predict_action_chunk_with_vector
    bool predict_action_chunk_with_emb(
        const uint8_t* h_agent_img,
        const uint8_t* h_wrist_img,
        const float* h_text_emb,
        float* h_out_actions,
        int num_euler_steps = 4,
        const float* custom_z0 = nullptr
    ) {
        return predict_action_chunk_with_vector(h_agent_img, h_wrist_img, h_text_emb, h_out_actions, num_euler_steps, custom_z0);
    }

    // Predicts 8x7 action chunk using a task string lookup
    bool predict_action_chunk(
        const uint8_t* h_agent_img,
        const uint8_t* h_wrist_img,
        const std::string& task_description,
        float* h_out_actions,
        int num_euler_steps = 4,
        const float* custom_z0 = nullptr
    );

    // Direct tensor-to-tensor forward pass for parity verification
    void forward_encoder(const float* d_obs_6x256x256, const float* d_text_emb_256, float* d_z_vis_16x256);
    void forward_decoder(const float* d_z_sigma_8x7, const float* d_z_vis_16x256, const float* d_cond_256, float* d_v_pred_8x7);
    void set_seed(unsigned int seed) { rng_.seed(seed); }

    float* get_tensor(const std::string& name);

    // Optimized Vision Encoder Buffers & Precomputed Weights
    cudaStream_t get_stream() const { return stream_; }
    float* get_gn_stats() const { return d_gn_stats_; }
    float* get_attn_wk_fused(int s) const { return d_attn_pool_wk_fused_[s]; }
    float* get_attn_bk_fused(int s) const { return d_attn_pool_bk_fused_[s]; }
    float* get_attn_wv_fused(int s) const { return d_attn_pool_wv_fused_[s]; }
    float* get_attn_bv_fused(int s) const { return d_attn_pool_bv_fused_[s]; }
    float* get_attn_q_head(int s) const { return d_attn_pool_q_head_[s]; }

private:
    int device_id_;
    cublasHandle_t cublas_handle_;
    cudaStream_t stream_;
    std::mt19937 rng_;
    std::normal_distribution<float> norm_dist_{0.0f, 1.0f};

    // Weight storage
    float* d_weights_buffer_ = nullptr;
    size_t total_weight_bytes_ = 0;
    std::unordered_map<std::string, TensorInfo> tensors_;

    // Task text embeddings lookup
    std::unordered_map<std::string, std::vector<float>> task_embeddings_;
    float* d_text_emb_ = nullptr;
    bool has_task_vector_ = false;

    // Action scale bounds
    float h_action_min_[7];
    float h_action_max_[7];
    float* d_action_min_ = nullptr;
    float* d_action_max_ = nullptr;

    // Static scratchpad device memory buffers (Zero allocations at runtime)
    uint8_t* d_raw_agent_ = nullptr;   // [256, 256, 3]
    uint8_t* d_raw_wrist_ = nullptr;   // [256, 256, 3]
    float* d_final_actions_ = nullptr;  // [8, 7]
    float* d_obs_ = nullptr;            // [6, 256, 256]
    float* d_z_vis_ = nullptr;          // [16, 256]
    float* d_z_sigma_ = nullptr;        // [8, 7]
    float* d_v_pred_ = nullptr;         // [8, 7]
    float* d_cond_ = nullptr;           // [256]
    float* d_t_emb_ = nullptr;          // [256]

    // Intermediate vision activations scratchpad
    float* d_vis_scratch_a_ = nullptr;  // Large scratchpad A
    float* d_vis_scratch_b_ = nullptr;  // Large scratchpad B
    float* d_vis_scratch_c_ = nullptr;  // Large scratchpad C
    float* d_vis_scratch_d_ = nullptr;  // Large scratchpad D
    float* d_im2col_ = nullptr;         // Convolution im2col scratchpad
    float* d_film_ss_ = nullptr;        // FiLM scale/shift scratchpad
    float* d_pool_k_ = nullptr;         // AttnPool K scratchpad
    float* d_pool_v_ = nullptr;         // AttnPool V scratchpad
    float* d_pool_k_head_ = nullptr;    // AttnPool K head scratchpad
    float* d_pool_v_head_ = nullptr;    // AttnPool V head scratchpad
    float* d_pool_q_ = nullptr;         // AttnPool Q scratchpad
    float* d_pool_q_head_ = nullptr;    // AttnPool Q head scratchpad
    float* d_pool_scores_ = nullptr;    // AttnPool scores scratchpad
    float* d_pool_ctx_head_ = nullptr;  // AttnPool context head scratchpad
    float* d_pool_context_ = nullptr;   // AttnPool context scratchpad
    float* d_gn_stats_ = nullptr;       // GroupNorm multi-block atomic reduction scratchpad (16 floats)

    // Precomputed / Fused AttnPool weights for stages 0..3 (96, 192, 192, 384 channels)
    float* d_attn_pool_wk_fused_[4] = {nullptr};
    float* d_attn_pool_bk_fused_[4] = {nullptr};
    float* d_attn_pool_wv_fused_[4] = {nullptr};
    float* d_attn_pool_bv_fused_[4] = {nullptr};
    float* d_attn_pool_q_head_[4] = {nullptr};

    // Transformer scratchpad
    float* d_trans_h_ = nullptr;        // [8, 256]
    float* d_trans_norm_ = nullptr;     // [8, 256]
    float* d_trans_qkv_ = nullptr;      // [8, 768]
    float* d_trans_q_head_ = nullptr;   // [8, 8, 32]
    float* d_trans_k_head_ = nullptr;   // [8, 16, 32]
    float* d_trans_v_head_ = nullptr;   // [8, 16, 32]
    float* d_trans_scores_ = nullptr;   // [8, 8, 16]
    float* d_trans_ctx_head_ = nullptr; // [8, 8, 32]
    float* d_trans_context_ = nullptr;  // [8, 256]
    float* d_trans_out_ = nullptr;      // [8, 256]
    float* d_trans_mlp_ = nullptr;      // [8, 1024]
    float* d_trans_film_ = nullptr;     // [512]
    float* d_temp_k_ = nullptr;         // [16, 256]
    float* d_temp_v_ = nullptr;         // [16, 256]

    // Precomputed Visual KV Cache buffers
    float* d_cached_k_ = nullptr;       // [4, 8, 16, 32]
    float* d_cached_v_ = nullptr;       // [4, 8, 16, 32]
    float* d_q_proj_ = nullptr;         // [8, 256]

    // Native CUDA Graph capture for ODE loop
    cudaGraph_t ode_graph_;
    cudaGraphExec_t ode_graph_exec_ = nullptr;
    bool graph_captured_ = false;

    friend class DiagnosticRunner;
    friend class Profiler;
    void precompute_kv_cache();
    void forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond);
    void forward_decoder_kv(const float* d_z_sigma_8x7, const float* d_cond_256, float* d_v_pred_8x7);
    void allocate_scratchpads();
    void free_scratchpads();
};