Upload folder using huggingface_hub
Browse files- .gitignore +37 -0
- README.md +292 -0
- action_scale.json +22 -0
- action_utils.py +32 -0
- c_engine/Makefile +46 -0
- c_engine/README.md +227 -0
- c_engine/build.sh +8 -0
- c_engine/canonical_tasks.json +42 -0
- c_engine/export_from_safetensors.py +169 -0
- c_engine/export_text_encoder_onnx.py +168 -0
- c_engine/infer_c.py +443 -0
- c_engine/kernels.cuh +282 -0
- c_engine/parvulus_c_api.cpp +101 -0
- c_engine/parvulus_c_api.h +125 -0
- c_engine/parvulus_engine.cu +686 -0
- c_engine/parvulus_engine.h +187 -0
- c_engine/parvulus_runtime_cli.cpp +334 -0
- c_engine/test_vision_encoder.cpp +75 -0
- c_engine/vision_encoder.cu +555 -0
- c_engine/vision_encoder.cuh +560 -0
- c_engine/vision_encoder.h +53 -0
- config.json +43 -0
- eval_benchmark_parallel.py +555 -0
- eval_libero.py +225 -0
- infer.py +297 -0
- model.safetensors +3 -0
- parvulus.py +664 -0
- parvulus.safetensors +3 -0
- parvulus_auxiliary.safetensors +3 -0
- parvulus_full.safetensors +3 -0
- parvulus_runtime.py +345 -0
- prepare_runtime.py +144 -0
- pyproject.toml +42 -0
- requirements.txt +9 -0
- requirements_eval.txt +15 -0
- test_continuous_pipelined.py +218 -0
- test_standalone_runtime.py +90 -0
- text_encoder.py +74 -0
.gitignore
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Byte-compiled / optimized / DLL files
|
| 2 |
+
__pycache__/
|
| 3 |
+
*.py[cod]
|
| 4 |
+
*$py.class
|
| 5 |
+
|
| 6 |
+
# C / C++ build objects
|
| 7 |
+
*.o
|
| 8 |
+
*.obj
|
| 9 |
+
|
| 10 |
+
# Dynamic & Static libraries & binary weights (built locally via scripts / make)
|
| 11 |
+
c_engine/*.o
|
| 12 |
+
c_engine/*.so
|
| 13 |
+
c_engine/*.bin
|
| 14 |
+
c_engine/parvulus_runtime
|
| 15 |
+
c_engine/test_vision_encoder
|
| 16 |
+
|
| 17 |
+
# Exported large ONNX models (generated on demand via converter or downloaded from HF)
|
| 18 |
+
*.onnx
|
| 19 |
+
c_engine/*.onnx
|
| 20 |
+
|
| 21 |
+
# Temporary runs, outputs, archives and logs
|
| 22 |
+
*.log
|
| 23 |
+
*.mp4
|
| 24 |
+
*.tar.gz
|
| 25 |
+
eval_results/
|
| 26 |
+
outputs/
|
| 27 |
+
MUJOCO_LOG.TXT
|
| 28 |
+
|
| 29 |
+
# Virtual environments
|
| 30 |
+
.venv/
|
| 31 |
+
env/
|
| 32 |
+
venv/
|
| 33 |
+
ENV/
|
| 34 |
+
|
| 35 |
+
# OS metadata
|
| 36 |
+
.DS_Store
|
| 37 |
+
Thumbs.db
|
README.md
CHANGED
|
@@ -1,3 +1,295 @@
|
|
| 1 |
---
|
|
|
|
|
|
|
| 2 |
license: apache-2.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
language:
|
| 3 |
+
- en
|
| 4 |
license: apache-2.0
|
| 5 |
+
tags:
|
| 6 |
+
- robotics
|
| 7 |
+
- flow-matching
|
| 8 |
+
- world-models
|
| 9 |
+
- imitation-learning
|
| 10 |
+
- libero
|
| 11 |
+
- action-chunking
|
| 12 |
+
- pytorch
|
| 13 |
+
- safetensors
|
| 14 |
+
pipeline_tag: robotics
|
| 15 |
---
|
| 16 |
+
|
| 17 |
+
# PARVULUS: 10M-Scale World Action Model for Robot Manipulation
|
| 18 |
+
|
| 19 |
+
**PARVULUS** is an ultra-efficient 10M-parameter Flow Matching World Action Model (WAM) designed for continuous robot manipulation. Trained on the canonical [LIBERO](https://github.com/Lifelong-Robot-Learning/LIBERO) benchmark, Parvulus achieves a **90.88% average success rate across all 40 tasks** while running at real-time control frequencies (>100 Hz in C++/CUDA, sub-40ms with PyTorch KV-caching).
|
| 20 |
+
|
| 21 |
+
---
|
| 22 |
+
|
| 23 |
+
## 1. Highlights
|
| 24 |
+
|
| 25 |
+
- **9.98M parameters** (~20 MB in BF16/FP16, ~38 MB in FP32) for standalone robot control.
|
| 26 |
+
- **90.88% $\pm$ 0.37%** average success across all 40 canonical LIBERO tasks ($N=6,000$ simulation rollouts).
|
| 27 |
+
- Pinned `google-t5/t5-small` text representations with a trained 2-layer projection head.
|
| 28 |
+
- **Multiple Execution Modes**:
|
| 29 |
+
- **PyTorch KV-Cached**: $\approx 38.8\,\text{ms}$ ($206\,\text{Hz}$ action bandwidth).
|
| 30 |
+
- **Standalone C++ / CUDA Engine**: $\mathbf{7.28\,\text{ms}}$ ($>1,000\,\text{Hz}$ action bandwidth, pure CUDA/cuBLAS).
|
| 31 |
+
- **Intel Iris Xe Hybrid**: $\approx 106.6\,\text{ms}$ on low-power laptop integrated graphics (`xpu` vision + CPU ODE).
|
| 32 |
+
- Includes optional auxiliary training decoders (`parvulus_auxiliary.safetensors`, 29.15M params) for auto-regressive visual self-rollout, motion change estimation, and self-reconstruction from noisy images.
|
| 33 |
+
|
| 34 |
+
---
|
| 35 |
+
|
| 36 |
+
## 2. Benchmark Performance
|
| 37 |
+
|
| 38 |
+
Evaluated across the 4 canonical LIBERO suites ($N=6,000$ physical rollouts, 10 seeds, receding-horizon chunking $H=8, C=8$):
|
| 39 |
+
|
| 40 |
+
| Benchmark Suite | Canonical Tasks | Rollout Trials | Success Rate |
|
| 41 |
+
| :--- | :---: | :---: | :---: |
|
| 42 |
+
| **LIBERO-Spatial** | 10 | 1,500 | **92.67%** |
|
| 43 |
+
| **LIBERO-Object** | 10 | 1,500 | **91.13%** |
|
| 44 |
+
| **LIBERO-Goal** | 10 | 1,500 | **89.47%** |
|
| 45 |
+
| **LIBERO-10 (Long)** | 10 | 1,500 | **90.27%** |
|
| 46 |
+
| **Full Benchmark Average** | **40** | **6,000** | **90.88% $\pm$ 0.37%** |
|
| 47 |
+
|
| 48 |
+
---
|
| 49 |
+
|
| 50 |
+
## 3. Repository Structure
|
| 51 |
+
|
| 52 |
+
```
|
| 53 |
+
.
|
| 54 |
+
├── parvulus.safetensors # Action-only core policy weights (9.98M params, ~39 MB FP32)
|
| 55 |
+
├── parvulus_full.safetensors # Full model weights: action policy + auxiliary heads (39.12M params, ~156 MB)
|
| 56 |
+
├── parvulus_auxiliary.safetensors # Auxiliary world model heads only (video, change, self-recon; 29.15M params)
|
| 57 |
+
├── model.safetensors # HuggingFace default entrypoint (alias of parvulus.safetensors)
|
| 58 |
+
│
|
| 59 |
+
├── config.json # Model hyperparameters and pinned T5 commit revision
|
| 60 |
+
├── action_scale.json # 7-DoF per-channel min/max normalization boundaries
|
| 61 |
+
├── pyproject.toml # Standard package configuration
|
| 62 |
+
├── requirements.txt # Minimal Python dependencies
|
| 63 |
+
├── prepare_runtime.py # Setup script: builds C engine, generates binaries, prepares text encoder
|
| 64 |
+
│
|
| 65 |
+
├── parvulus_runtime.py # High-performance Python runtime for standalone C engine
|
| 66 |
+
├── infer.py # High-level ParvulusInference client (supports CUDA, C engine, XPU, Hybrid)
|
| 67 |
+
├── parvulus.py # Complete PyTorch architecture (DualViewEncoder, CVMixer, FlowDecoder)
|
| 68 |
+
├── text_encoder.py # Pinned T5 text tokenizer and projection head
|
| 69 |
+
├── action_utils.py # Action normalization and denormalization helpers
|
| 70 |
+
│
|
| 71 |
+
└── c_engine/ # Dependency-free C++ / CUDA Standalone Engine
|
| 72 |
+
├── Makefile # Make-based build script
|
| 73 |
+
├── build.sh # Convenient build script
|
| 74 |
+
├── parvulus_engine.cu / .h # Core C++/CUDA inference engine (136 Hz chunk rate)
|
| 75 |
+
├── vision_encoder.cu / .cuh # Fused DualView vision encoder
|
| 76 |
+
├── parvulus_c_api.cpp / .h # Standard C ABI exports for C/C++/ROS 2/Rust
|
| 77 |
+
├── parvulus_runtime_cli.cpp # Standalone CLI binary entry point
|
| 78 |
+
├── canonical_tasks.json # 40 canonical task instructions
|
| 79 |
+
├── export_from_safetensors.py # Script generating model binary, scale, and embeddings
|
| 80 |
+
└── export_text_encoder_onnx.py # Standalone converter for T5-small text encoder to ONNX
|
| 81 |
+
```
|
| 82 |
+
|
| 83 |
+
---
|
| 84 |
+
|
| 85 |
+
## 4. Quickstart: Python Inference
|
| 86 |
+
|
| 87 |
+
### Installation
|
| 88 |
+
|
| 89 |
+
```bash
|
| 90 |
+
pip install -r requirements.txt
|
| 91 |
+
```
|
| 92 |
+
|
| 93 |
+
### 1. Basic Action Chunk Prediction
|
| 94 |
+
|
| 95 |
+
```python
|
| 96 |
+
import numpy as np
|
| 97 |
+
from PIL import Image
|
| 98 |
+
from infer import ParvulusInference
|
| 99 |
+
|
| 100 |
+
# Initialize client (auto-detects CUDA / XPU / CPU; enables fast KV-cache by default)
|
| 101 |
+
infer = ParvulusInference.from_pretrained(
|
| 102 |
+
".",
|
| 103 |
+
device="cuda", # Options: 'cuda', 'cpu', 'xpu', 'hybrid', or 'auto'
|
| 104 |
+
use_kv_cache=True, # Enables sub-40ms inference
|
| 105 |
+
num_ode_steps=10
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
# Prepare inputs: dual 256x256 RGB cameras + text prompt
|
| 109 |
+
agentview_img = np.array(Image.open("agentview.png")) # [256, 256, 3] uint8
|
| 110 |
+
wristview_img = np.array(Image.open("wristview.png")) # [256, 256, 3] uint8
|
| 111 |
+
task_prompt = "pick up the black bowl in the center and place it on the plate"
|
| 112 |
+
|
| 113 |
+
# Predict 8-step continuous action chunk [8, 7]
|
| 114 |
+
# (dx, dy, dz, drot_x, drot_y, drot_z, gripper)
|
| 115 |
+
action_chunk = infer.predict_chunk(agentview_img, wristview_img, task_description=task_prompt)
|
| 116 |
+
print(f"Predicted action chunk (shape {action_chunk.shape}):\n{action_chunk}")
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
### 2. Environment Stepping Loop (Receding Horizon)
|
| 120 |
+
|
| 121 |
+
```python
|
| 122 |
+
infer.reset(task_description="pick up the milk and place it in the basket")
|
| 123 |
+
obs = env.reset()
|
| 124 |
+
agentview_img = obs["agentview_image"]
|
| 125 |
+
wristview_img = obs["robot0_eye_in_hand_image"]
|
| 126 |
+
|
| 127 |
+
done = False
|
| 128 |
+
while not done:
|
| 129 |
+
# In your simulation or real robot loop:
|
| 130 |
+
action = infer.step(agentview_img, wristview_img) # [7] float32 action
|
| 131 |
+
obs, reward, done, info = env.step(action)
|
| 132 |
+
agentview_img = obs["agentview_image"]
|
| 133 |
+
wristview_img = obs["robot0_eye_in_hand_image"]
|
| 134 |
+
```
|
| 135 |
+
|
| 136 |
+
### 3. Continuous Overlapped Pipelined Execution ($H=8, C=4$)
|
| 137 |
+
|
| 138 |
+
To achieve zero-latency continuous robot motion, Parvulus supports overlapping chunk generation with robot motion execution:
|
| 139 |
+
1. Predict an 8-step chunk ($A_0$).
|
| 140 |
+
2. Execute the first 4 actions ($A_0[0:4]$).
|
| 141 |
+
3. At step 4, capture a fresh observation and trigger background inference for the next chunk ($A_1$).
|
| 142 |
+
4. Concurrently execute the remaining tail actions ($A_0[4:8]$) on the robot while the model predicts $A_1$.
|
| 143 |
+
5. Transition seamlessly into $A_1$ with zero robot idling.
|
| 144 |
+
|
| 145 |
+
```python
|
| 146 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 147 |
+
|
| 148 |
+
executor = ThreadPoolExecutor(max_workers=1)
|
| 149 |
+
infer.reset(task_description="pick up the black bowl in the center and place it on the plate")
|
| 150 |
+
|
| 151 |
+
# 1. Reset environment and capture initial observation at t = 0
|
| 152 |
+
obs = env.reset()
|
| 153 |
+
agentview_img = obs["agentview_image"]
|
| 154 |
+
wristview_img = obs["robot0_eye_in_hand_image"]
|
| 155 |
+
|
| 156 |
+
# 2. Generate initial 8-step action chunk (covers t = 0..7)
|
| 157 |
+
current_chunk = infer.predict_chunk(agentview_img, wristview_img)
|
| 158 |
+
|
| 159 |
+
# 3. Execute the initial head (t = 0..3) before entering the pipelined loop
|
| 160 |
+
done = False
|
| 161 |
+
for a in current_chunk[0:4]:
|
| 162 |
+
obs, reward, done, _ = env.step(a)
|
| 163 |
+
if done:
|
| 164 |
+
break
|
| 165 |
+
|
| 166 |
+
# Continuous overlapped loop: each cycle executes tail actions (4..7) while predicting the next chunk
|
| 167 |
+
while not done:
|
| 168 |
+
# 4. Capture observation at step t = 4 (or 4k)
|
| 169 |
+
agentview_img = obs["agentview_image"]
|
| 170 |
+
wristview_img = obs["robot0_eye_in_hand_image"]
|
| 171 |
+
|
| 172 |
+
# 5. Trigger background inference for next chunk (covers upcoming horizon)
|
| 173 |
+
future_chunk = executor.submit(infer.predict_chunk, agentview_img, wristview_img)
|
| 174 |
+
|
| 175 |
+
# 6. Overlap: Execute remaining 4 tail actions (4..7) while background inference runs
|
| 176 |
+
for a in current_chunk[4:8]:
|
| 177 |
+
obs, reward, done, _ = env.step(a)
|
| 178 |
+
if done:
|
| 179 |
+
break
|
| 180 |
+
if done:
|
| 181 |
+
break
|
| 182 |
+
|
| 183 |
+
# 7. Retrieve newly generated chunk with zero robot idle time
|
| 184 |
+
current_chunk = future_chunk.result()
|
| 185 |
+
```
|
| 186 |
+
|
| 187 |
+
### 4. World-Model Self-Rollout (Mental Simulation)
|
| 188 |
+
|
| 189 |
+
Parvulus provides optional auxiliary decoders (`parvulus_auxiliary.safetensors`) for predicting future frames without interacting with the physical environment:
|
| 190 |
+
|
| 191 |
+
```python
|
| 192 |
+
infer = ParvulusInference.from_pretrained(
|
| 193 |
+
".",
|
| 194 |
+
load_auxiliary=True,
|
| 195 |
+
device="cuda"
|
| 196 |
+
)
|
| 197 |
+
|
| 198 |
+
# Predict action chunk and imagined next observation
|
| 199 |
+
action_chunk, next_agent_img, next_wrist_img = infer.rollout_step(
|
| 200 |
+
agentview_img, wristview_img, task_description="push the button"
|
| 201 |
+
)
|
| 202 |
+
```
|
| 203 |
+
|
| 204 |
+
---
|
| 205 |
+
|
| 206 |
+
## 5. Standalone Native C++ / CUDA Runtime (Vector-First, 7.3 ms / chunk)
|
| 207 |
+
|
| 208 |
+
For high-frequency physical robot deployments or low-overhead environments, Parvulus includes a standalone C++/CUDA runtime engine in `c_engine/` with **few dependencies**.
|
| 209 |
+
|
| 210 |
+
### Features
|
| 211 |
+
- Complete dual-camera visual encoding + 10-step Euler ODE flow matching loop runs in **<8 ms** (>1,000 Hz action bandwidth).
|
| 212 |
+
- Natively conditioned on 256-dim continuous embedding vectors (`task_vector`) to eliminate text tokenization latency in the robot control loop.
|
| 213 |
+
- Automatically maps arbitrary natural language instructions or perturbations into 256-dim vectors via ONNX Runtime (<4 ms) before passing to the engine.
|
| 214 |
+
|
| 215 |
+
### Preparing & Building the Runtime
|
| 216 |
+
```bash
|
| 217 |
+
# Automated preparation (builds C engine, prepares weights & ONNX text encoder)
|
| 218 |
+
python3 prepare_runtime.py
|
| 219 |
+
|
| 220 |
+
# Or build C++ engine directly:
|
| 221 |
+
cd c_engine && make -j$(nproc)
|
| 222 |
+
```
|
| 223 |
+
This produces:
|
| 224 |
+
- `libparvulus.so`: Shared C ABI library for Python ctypes, C, C++, ROS 2, or Rust.
|
| 225 |
+
- `parvulus_runtime`: Standalone CLI binary for headless benchmarks and batch action generation.
|
| 226 |
+
- `export_text_encoder_onnx.py`: Converter utility to export the 136 MB ONNX text encoder on demand (or download it from Hugging Face).
|
| 227 |
+
|
| 228 |
+
### Standalone CLI Benchmark
|
| 229 |
+
```bash
|
| 230 |
+
./c_engine/parvulus_runtime --benchmark 100
|
| 231 |
+
```
|
| 232 |
+
|
| 233 |
+
### Python Quickstart (`parvulus_runtime.py`)
|
| 234 |
+
```python
|
| 235 |
+
import numpy as np
|
| 236 |
+
from parvulus_runtime import ParvulusRuntime
|
| 237 |
+
|
| 238 |
+
# Initialize standalone C++/CUDA engine
|
| 239 |
+
runtime = ParvulusRuntime(device_id=0, num_euler_steps=10)
|
| 240 |
+
|
| 241 |
+
# 1. Condition with a 256-dim task vector:
|
| 242 |
+
runtime.reset(task_vector=my_256_dim_vector)
|
| 243 |
+
|
| 244 |
+
# Or condition with arbitrary text (automatic dynamic ONNX encoding):
|
| 245 |
+
# runtime.reset(task_description="grasp the black bowl and place it on the plate")
|
| 246 |
+
|
| 247 |
+
# 2. Step in real-time robot loop (streams only camera frames at 136 Hz):
|
| 248 |
+
action = runtime.step(agentview_img, wristview_img) # [7] float32
|
| 249 |
+
```
|
| 250 |
+
|
| 251 |
+
For detailed C ABI signatures and non-Python integration instructions, see [`c_engine/README.md`](c_engine/README.md).
|
| 252 |
+
|
| 253 |
+
---
|
| 254 |
+
|
| 255 |
+
## 6. Intel Iris Xe / Arc Heterogeneous Hybrid Mode
|
| 256 |
+
|
| 257 |
+
For deployment on mobile workstations or edge devices with Intel Iris Xe or Arc integrated graphics:
|
| 258 |
+
|
| 259 |
+
```python
|
| 260 |
+
infer = ParvulusInference.from_pretrained(".", device="hybrid")
|
| 261 |
+
```
|
| 262 |
+
|
| 263 |
+
In `hybrid` mode:
|
| 264 |
+
1. Perception backbone (`DualViewEncoder`) executes on the iGPU (`torch.xpu`).
|
| 265 |
+
2. Only 16 pooled visual tokens (16 KB) transfer over PCIe to the host CPU.
|
| 266 |
+
3. The 10-step ODE Transformer loop runs on the high-clock CPU cores.
|
| 267 |
+
|
| 268 |
+
---
|
| 269 |
+
|
| 270 |
+
## 7. Model Details & Pinned Dependencies
|
| 271 |
+
|
| 272 |
+
- **Input Specifications**:
|
| 273 |
+
- `agentview`: $[256, 256, 3]$ RGB.
|
| 274 |
+
- `wristview`: $[256, 256, 3]$ RGB.
|
| 275 |
+
- `instruction`: Natural language string.
|
| 276 |
+
- **Output Action Format**:
|
| 277 |
+
- 8-step continuous trajectory $\mathbf{A}_t \in \mathbb{R}^{8 \times 7}$ ($dx, dy, dz, d\theta_x, d\theta_y, d\theta_z, \text{gripper}$).
|
| 278 |
+
- **Pinned Text Model**:
|
| 279 |
+
- `google-t5/t5-small` pinned to commit revision `df1b051c49625cf57a3d0d8d3863ed4d13564fe4`.
|
| 280 |
+
|
| 281 |
+
---
|
| 282 |
+
|
| 283 |
+
## Citation
|
| 284 |
+
Parvulus model weights and the associated inference code are released under a permissive license. If you find our work useful, please cite:
|
| 285 |
+
|
| 286 |
+
```bibtex
|
| 287 |
+
@inproceedings{
|
| 288 |
+
sivas2026parvulus,
|
| 289 |
+
title={Parvulus: A 10M-Scale World Action Model for Robot Manipulation},
|
| 290 |
+
author={Abdullah Ali Sivas and Peter Cho-Ho Lam and Zirui Zhou and Yong Zhang},
|
| 291 |
+
booktitle={CoRL 2026 Workshop on The Action Gap: Large Generative Models Learn Rich Knowledge},
|
| 292 |
+
year={2026},
|
| 293 |
+
url={https://openreview.net/forum?id=VcS0dE1Aoz}
|
| 294 |
+
}
|
| 295 |
+
```
|
action_scale.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"min": [
|
| 3 |
+
-0.9375,
|
| 4 |
+
-0.9375,
|
| 5 |
+
-0.9375,
|
| 6 |
+
-0.2582142949104309,
|
| 7 |
+
-0.375,
|
| 8 |
+
-0.3675000071525574,
|
| 9 |
+
-1.0
|
| 10 |
+
],
|
| 11 |
+
"max": [
|
| 12 |
+
0.9375,
|
| 13 |
+
0.9375,
|
| 14 |
+
0.9375,
|
| 15 |
+
0.3557142913341522,
|
| 16 |
+
0.375,
|
| 17 |
+
0.375,
|
| 18 |
+
1.0
|
| 19 |
+
],
|
| 20 |
+
"source_stats_file": "data/libero_hf/meta/stats.json",
|
| 21 |
+
"note": "dx, dy, dz, drot_x, drot_y, drot_z, gripper -- per-dimension min/max over the full dataset"
|
| 22 |
+
}
|
action_utils.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Action normalization utilities for the 7-DoF LIBERO action space:
|
| 2 |
+
(dx, dy, dz, drot_x, drot_y, drot_z, gripper).
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Tuple
|
| 8 |
+
import torch
|
| 9 |
+
|
| 10 |
+
DEFAULT_SCALE_PATH = Path(__file__).resolve().parent / "action_scale.json"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def load_action_scale(path: str | Path = DEFAULT_SCALE_PATH) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 14 |
+
"""Loads per-dimension min/max normalization limits."""
|
| 15 |
+
path = Path(path)
|
| 16 |
+
with open(path, "r", encoding="utf-8") as f:
|
| 17 |
+
data = json.load(f)
|
| 18 |
+
action_min = torch.tensor(data["min"], dtype=torch.float32)
|
| 19 |
+
action_max = torch.tensor(data["max"], dtype=torch.float32)
|
| 20 |
+
return action_min, action_max
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def normalize_action(raw: torch.Tensor, min_: torch.Tensor, max_: torch.Tensor) -> torch.Tensor:
|
| 24 |
+
"""Normalizes raw actions to [-1, 1]."""
|
| 25 |
+
span = (max_ - min_).clamp(min=1e-8)
|
| 26 |
+
return 2.0 * (raw - min_) / span - 1.0
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def denormalize_action(scaled: torch.Tensor, min_: torch.Tensor, max_: torch.Tensor) -> torch.Tensor:
|
| 30 |
+
"""Denormalizes actions from [-1, 1] back to physical robot units."""
|
| 31 |
+
span = max_ - min_
|
| 32 |
+
return (scaled + 1.0) / 2.0 * span + min_
|
c_engine/Makefile
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
CUDA_PATH ?= /usr/local/cuda
|
| 2 |
+
ifeq ($(wildcard $(CUDA_PATH)/bin/nvcc),)
|
| 3 |
+
CUDA_PATH := /usr
|
| 4 |
+
endif
|
| 5 |
+
|
| 6 |
+
NVCC := $(CUDA_PATH)/bin/nvcc
|
| 7 |
+
CXX := g++
|
| 8 |
+
|
| 9 |
+
NVCC_FLAGS := -O3 -c -Xcompiler -fPIC -std=c++17 -I. -I$(CUDA_PATH)/include
|
| 10 |
+
CXX_FLAGS := -O3 -fPIC -std=c++17 -I. -I$(CUDA_PATH)/include
|
| 11 |
+
LDFLAGS := -L$(CUDA_PATH)/lib64 -L. -lcublas -lcudart -Wl,-rpath,'$$ORIGIN' -Wl,-rpath,$(CUDA_PATH)/lib64
|
| 12 |
+
|
| 13 |
+
TARGET_LIB := libparvulus.so
|
| 14 |
+
TARGET_CLI := parvulus_runtime
|
| 15 |
+
TARGET_TEST := test_vision_encoder
|
| 16 |
+
|
| 17 |
+
.PHONY: all clean distclean test weights
|
| 18 |
+
|
| 19 |
+
all: $(TARGET_LIB) $(TARGET_CLI) $(TARGET_TEST)
|
| 20 |
+
|
| 21 |
+
weights:
|
| 22 |
+
@if [ ! -f parvulus_model_weights.bin ] || [ ! -f action_scale.bin ] || [ ! -f task_embeddings.bin ]; then \
|
| 23 |
+
python3 export_from_safetensors.py; \
|
| 24 |
+
fi
|
| 25 |
+
|
| 26 |
+
parvulus_engine.o: parvulus_engine.cu parvulus_engine.h vision_encoder.cu vision_encoder.cuh vision_encoder.h kernels.cuh
|
| 27 |
+
$(NVCC) $(NVCC_FLAGS) parvulus_engine.cu -o parvulus_engine.o
|
| 28 |
+
|
| 29 |
+
$(TARGET_LIB): parvulus_engine.o parvulus_c_api.cpp parvulus_c_api.h
|
| 30 |
+
$(CXX) -shared -fPIC parvulus_engine.o parvulus_c_api.cpp -o $(TARGET_LIB) $(CXX_FLAGS) $(LDFLAGS)
|
| 31 |
+
|
| 32 |
+
$(TARGET_CLI): parvulus_runtime_cli.cpp $(TARGET_LIB)
|
| 33 |
+
$(CXX) $(CXX_FLAGS) parvulus_runtime_cli.cpp -o $(TARGET_CLI) -lparvulus $(LDFLAGS)
|
| 34 |
+
|
| 35 |
+
$(TARGET_TEST): test_vision_encoder.cpp $(TARGET_LIB)
|
| 36 |
+
$(CXX) $(CXX_FLAGS) test_vision_encoder.cpp -o $(TARGET_TEST) -lparvulus $(LDFLAGS)
|
| 37 |
+
|
| 38 |
+
test: all weights
|
| 39 |
+
./$(TARGET_TEST)
|
| 40 |
+
./$(TARGET_CLI) --benchmark 20
|
| 41 |
+
|
| 42 |
+
clean:
|
| 43 |
+
rm -f parvulus_engine.o $(TARGET_LIB) $(TARGET_CLI) $(TARGET_TEST)
|
| 44 |
+
|
| 45 |
+
distclean: clean
|
| 46 |
+
rm -f parvulus_model_weights.bin action_scale.bin task_embeddings.bin *.onnx
|
c_engine/README.md
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PARVULUS Standalone Native Runtime Engine (C++ / CUDA)
|
| 2 |
+
|
| 3 |
+
High-performance, pure CUDA/C++ inference runtime for the **PARVULUS** 10M-parameter Flow Matching World Action Model.
|
| 4 |
+
|
| 5 |
+
- Built solely with `nvcc`, `g++`, `libcublas`, and `libcudart`.
|
| 6 |
+
- Policy natively accepts 256-dimensional continuous conditioning vectors ($[256]\text{ float32}$). Needs to be evaluated once per episode or can be modified asynchronously.
|
| 7 |
+
- **$\mathbf{\lt 8\,\text{ms}}$ per 8-step action chunk** (~$\gt 125\,\text{Hz}$ chunk rate, $>1,000\,\text{Hz}$ action bandwidth).
|
| 8 |
+
- Direct C ABI shared library (`libparvulus.so`), standalone CLI binary (`parvulus_runtime`), and zero-overhead Python bindings.
|
| 9 |
+
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
## 1. Directory Structure
|
| 13 |
+
|
| 14 |
+
```
|
| 15 |
+
c_engine/
|
| 16 |
+
├── Makefile # Make-based build system for library, CLI, and test binaries
|
| 17 |
+
├── build.sh # Build script (runs make clean && make -j)
|
| 18 |
+
│
|
| 19 |
+
├── parvulus_engine.h # Core C++ ParvulusEngine class header
|
| 20 |
+
├── parvulus_engine.cu # Pure CUDA/cuBLAS implementation (Euler ODE loop, cross-attention)
|
| 21 |
+
├── vision_encoder.h # DualView visual backbone header
|
| 22 |
+
├── vision_encoder.cuh # DualView CUDA kernels (patch embed, depthwise-separable convs)
|
| 23 |
+
├── vision_encoder.cu # DualView CUDA implementation
|
| 24 |
+
├── kernels.cuh # Fused CUDA kernels (LayerNorm, SiLU, Softmax, Add, Unscale)
|
| 25 |
+
│
|
| 26 |
+
├── parvulus_c_api.h # Standard C ABI header for external language bindings / middleware
|
| 27 |
+
├── parvulus_c_api.cpp # C ABI export implementation
|
| 28 |
+
│
|
| 29 |
+
├── parvulus_runtime_cli.cpp # Standalone CLI binary entry point
|
| 30 |
+
├── test_vision_encoder.cpp # Standalone numerical parity test for vision backbone
|
| 31 |
+
├── infer_c.py # Python ctypes wrapper
|
| 32 |
+
│
|
| 33 |
+
├── export_from_safetensors.py # Serializes .safetensors into binary weight files
|
| 34 |
+
├── export_text_encoder_onnx.py # Standalone converter for T5 + projection head to ONNX
|
| 35 |
+
├── parvulus_model_weights.bin # Exported FP32 weights (37.28 MB, 273 tensors)
|
| 36 |
+
├── task_embeddings.bin # Pre-cached 256-dim embeddings for canonical tasks
|
| 37 |
+
└── action_scale.bin # 7-DoF min/max action normalization bounds
|
| 38 |
+
```
|
| 39 |
+
|
| 40 |
+
---
|
| 41 |
+
|
| 42 |
+
## 2. Requirements & Building
|
| 43 |
+
|
| 44 |
+
### Prerequisites
|
| 45 |
+
- **CUDA Toolkit** (11.8+ or 12.0+) with `nvcc`, `libcudart`, `libcublas`
|
| 46 |
+
- **C++17 Compiler** (`g++` 9.0+)
|
| 47 |
+
- *Python or PyTorch runtime are not needed for C/C++ usage.*
|
| 48 |
+
|
| 49 |
+
### Build Targets
|
| 50 |
+
|
| 51 |
+
To build the shared library `libparvulus.so`, standalone CLI binary `parvulus_runtime`, and test binary:
|
| 52 |
+
|
| 53 |
+
```bash
|
| 54 |
+
cd c_engine
|
| 55 |
+
make -j$(nproc)
|
| 56 |
+
```
|
| 57 |
+
|
| 58 |
+
Or run the build script:
|
| 59 |
+
```bash
|
| 60 |
+
./build.sh
|
| 61 |
+
```
|
| 62 |
+
|
| 63 |
+
### Run Sanity Verification Tests & Benchmark:
|
| 64 |
+
```bash
|
| 65 |
+
make test
|
| 66 |
+
```
|
| 67 |
+
This runs:
|
| 68 |
+
1. `test_vision_encoder`: Verifies DualView vision backbone parity and reports perception latency.
|
| 69 |
+
2. `parvulus_runtime --benchmark 20`: Runs a 20-chunk end-to-end benchmark and verifies end-to-end latency.
|
| 70 |
+
|
| 71 |
+
---
|
| 72 |
+
|
| 73 |
+
## 3. Standalone CLI Binary (`parvulus_runtime`)
|
| 74 |
+
|
| 75 |
+
The binary `parvulus_runtime` can be executed directly from bash or integrated into non-Python robotics controllers.
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
./c_engine/parvulus_runtime [options]
|
| 79 |
+
```
|
| 80 |
+
|
| 81 |
+
### Command-Line Options
|
| 82 |
+
|
| 83 |
+
| Option | Type | Default | Description |
|
| 84 |
+
| :--- | :--- | :--- | :--- |
|
| 85 |
+
| `--device` | `int` | `0` | Target CUDA GPU device ID |
|
| 86 |
+
| `--weights` | `path` | `parvulus_model_weights.bin` | Path to binary model weights |
|
| 87 |
+
| `--embeddings` | `path` | `task_embeddings.bin` | Optional lookup table for canonical tasks |
|
| 88 |
+
| `--scale` | `path` | `action_scale.bin` | Action scale boundaries |
|
| 89 |
+
| `--nfe` | `int` | `10` | Euler ODE flow steps (10 recommended) |
|
| 90 |
+
| `--task_vector_file` | `path` | `""` | Load 256-dim task vector from `.bin`, `.npy`, or `.txt` |
|
| 91 |
+
| `--task_vector` | `string` | `""` | Comma-separated 256 floats: `"0.12,-0.05,..."` |
|
| 92 |
+
| `--task` | `string` | `""` | Task string (resolved via pre-cached table) |
|
| 93 |
+
| `--agent_img` | `path` | `""` | Path to raw $256 \times 256 \times 3$ uint8 RGB agent camera frame |
|
| 94 |
+
| `--wrist_img` | `path` | `""` | Path to raw $256 \times 256 \times 3$ uint8 RGB wrist camera frame |
|
| 95 |
+
| `--output` | `path` | `""` | File path to write predicted $8 \times 7$ action trajectory |
|
| 96 |
+
| `--benchmark` | `int` | `0` | Run $N$ benchmark iterations and print latency breakdown |
|
| 97 |
+
|
| 98 |
+
### CLI Usage Examples
|
| 99 |
+
|
| 100 |
+
**1. High-frequency Latency Benchmark:**
|
| 101 |
+
```bash
|
| 102 |
+
./c_engine/parvulus_runtime --benchmark 100
|
| 103 |
+
```
|
| 104 |
+
*Output:*
|
| 105 |
+
```
|
| 106 |
+
[Benchmark] Completed 100 chunks in 727.5 ms
|
| 107 |
+
[Benchmark] Mean Latency: 7.275 ms / chunk
|
| 108 |
+
[Benchmark] Action Rate: 1099.7 Hz (8 actions / 7.275 ms)
|
| 109 |
+
```
|
| 110 |
+
|
| 111 |
+
**2. Predict Actions from a Custom Task Vector File:**
|
| 112 |
+
```bash
|
| 113 |
+
./c_engine/parvulus_runtime \
|
| 114 |
+
--task_vector_file custom_task.bin \
|
| 115 |
+
--agent_img agentview.raw \
|
| 116 |
+
--wrist_img wristview.raw \
|
| 117 |
+
--output predicted_actions.txt
|
| 118 |
+
```
|
| 119 |
+
|
| 120 |
+
---
|
| 121 |
+
|
| 122 |
+
## 4. C ABI Interface (`parvulus_c_api.h`)
|
| 123 |
+
|
| 124 |
+
For embedding into C, C++, ROS 2 nodes, or foreign languages (Rust, Go, C#), include `parvulus_c_api.h`:
|
| 125 |
+
|
| 126 |
+
```c
|
| 127 |
+
#include "parvulus_c_api.h"
|
| 128 |
+
|
| 129 |
+
// 1. Initialize engine on GPU 0
|
| 130 |
+
ParvulusHandle engine = parvulus_create(0);
|
| 131 |
+
|
| 132 |
+
// 2. Load model weights and action normalizer
|
| 133 |
+
parvulus_load(engine, "parvulus_model_weights.bin", "task_embeddings.bin", "action_scale.bin");
|
| 134 |
+
|
| 135 |
+
// 3. Set episode task conditioning vector (256 floats)
|
| 136 |
+
float task_vector[256];
|
| 137 |
+
load_my_task_vector(task_vector);
|
| 138 |
+
parvulus_set_task_vector(engine, task_vector);
|
| 139 |
+
|
| 140 |
+
// 4. In robot real-time loop:
|
| 141 |
+
uint8_t agentview_rgb[256 * 256 * 3];
|
| 142 |
+
uint8_t wristview_rgb[256 * 256 * 3];
|
| 143 |
+
float action_chunk[8 * 7]; // 8 timesteps x 7-DoF actions
|
| 144 |
+
|
| 145 |
+
// Pass NULL for task_description to use the active task vector in GPU memory
|
| 146 |
+
parvulus_predict_chunk(engine, agentview_rgb, wristview_rgb, NULL, action_chunk, 10, NULL);
|
| 147 |
+
|
| 148 |
+
// 5. Cleanup
|
| 149 |
+
parvulus_destroy(engine);
|
| 150 |
+
```
|
| 151 |
+
|
| 152 |
+
### Exported Functions
|
| 153 |
+
|
| 154 |
+
```c
|
| 155 |
+
// Create / Destroy
|
| 156 |
+
ParvulusHandle parvulus_create(int device_id);
|
| 157 |
+
void parvulus_destroy(ParvulusHandle handle);
|
| 158 |
+
|
| 159 |
+
// Model Weights Loading
|
| 160 |
+
bool parvulus_load(ParvulusHandle handle, const char* weights_path, const char* embeddings_path, const char* scale_path);
|
| 161 |
+
|
| 162 |
+
// Task Conditioning (Vector-First)
|
| 163 |
+
bool parvulus_set_task_vector(ParvulusHandle handle, const float* task_vector_256);
|
| 164 |
+
bool parvulus_set_task_string(ParvulusHandle handle, const char* task_description);
|
| 165 |
+
bool parvulus_register_task_embedding(ParvulusHandle handle, const char* task_description, const float* embedding_256);
|
| 166 |
+
|
| 167 |
+
// Chunk Prediction
|
| 168 |
+
bool parvulus_predict_chunk(ParvulusHandle handle, const uint8_t* agentview_rgb, const uint8_t* wristview_rgb,
|
| 169 |
+
const char* task_description, float* out_actions_56, int num_euler_steps,
|
| 170 |
+
const float* custom_z0);
|
| 171 |
+
|
| 172 |
+
bool parvulus_predict_chunk_with_vector(ParvulusHandle handle, const uint8_t* agentview_rgb, const uint8_t* wristview_rgb,
|
| 173 |
+
const float* task_vector_256, float* out_actions_56, int num_euler_steps,
|
| 174 |
+
const float* custom_z0);
|
| 175 |
+
```
|
| 176 |
+
|
| 177 |
+
---
|
| 178 |
+
|
| 179 |
+
## 5. Python Integration
|
| 180 |
+
|
| 181 |
+
### Quickstart with `ParvulusRuntime`
|
| 182 |
+
|
| 183 |
+
```python
|
| 184 |
+
import numpy as np
|
| 185 |
+
from parvulus_runtime import ParvulusRuntime
|
| 186 |
+
|
| 187 |
+
# Initialize runtime
|
| 188 |
+
runtime = ParvulusRuntime(device_id=0, num_euler_steps=10)
|
| 189 |
+
|
| 190 |
+
# Mode A: Vector-First conditioning (e.g., custom, perturbed, or retrieved embedding)
|
| 191 |
+
custom_vector = np.random.randn(256).astype(np.float32)
|
| 192 |
+
runtime.reset(task_vector=custom_vector)
|
| 193 |
+
|
| 194 |
+
# Mode B: Dynamic text instruction (automatically handles perturbations via ONNX)
|
| 195 |
+
runtime.reset(task_description="grasp the black bowl and set it onto the plate")
|
| 196 |
+
|
| 197 |
+
# Real-time robot step loop (Receding Horizon, 136 Hz chunking)
|
| 198 |
+
agent_rgb = np.zeros((256, 256, 3), dtype=np.uint8)
|
| 199 |
+
wrist_rgb = np.zeros((256, 256, 3), dtype=np.uint8)
|
| 200 |
+
|
| 201 |
+
# Single 7-DoF step
|
| 202 |
+
action = runtime.step(agent_rgb, wrist_rgb)
|
| 203 |
+
|
| 204 |
+
# Or full 8-step chunk prediction [8, 7]
|
| 205 |
+
chunk = runtime.predict_chunk(agent_rgb, wrist_rgb)
|
| 206 |
+
```
|
| 207 |
+
|
| 208 |
+
---
|
| 209 |
+
|
| 210 |
+
## 6. Language Handling
|
| 211 |
+
|
| 212 |
+
When using string instructions:
|
| 213 |
+
1. If the instruction matches a canonical benchmark task, its embedding is resolved instantly ($<1\,\mu\text{s}$) from `task_embeddings.bin`.
|
| 214 |
+
2. If an instruction differs (e.g., *"grasp the black bowl between the plate and the ramekin"* vs. *"take the black bowl and set it onto the plate"*):
|
| 215 |
+
- If `parvulus_text_encoder.onnx` is present (or downloaded from Hugging Face / generated via converter), `ParvulusRuntime` encodes the prompt.
|
| 216 |
+
- If ONNX is not generated, `ParvulusRuntime` automatically uses PyTorch (`text_encoder.py`) without failure.
|
| 217 |
+
3. Task conditioning is set once per episode ($t=0$). The control loop streams only camera frames, completely bypassing any text-processing overhead.
|
| 218 |
+
|
| 219 |
+
### Converting or Downloading the ONNX Text Encoder
|
| 220 |
+
The ONNX text encoder binary is not tracked. To prepare it:
|
| 221 |
+
```bash
|
| 222 |
+
# Option A: Convert locally from base weights + T5-small
|
| 223 |
+
python3 c_engine/export_text_encoder_onnx.py
|
| 224 |
+
|
| 225 |
+
# Option B: Run automatic runtime preparation (downloads from Hugging Face or converts locally)
|
| 226 |
+
python3 prepare_runtime.py [--hf_repo <user/repo>]
|
| 227 |
+
```
|
c_engine/build.sh
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -e
|
| 3 |
+
|
| 4 |
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
| 5 |
+
cd "${SCRIPT_DIR}"
|
| 6 |
+
|
| 7 |
+
make clean
|
| 8 |
+
make -j"$(nproc)" all
|
c_engine/canonical_tasks.json
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
"open the middle drawer of the cabinet",
|
| 3 |
+
"open the top drawer and put the bowl inside",
|
| 4 |
+
"pick up the alphabet soup and place it in the basket",
|
| 5 |
+
"pick up the bbq sauce and place it in the basket",
|
| 6 |
+
"pick up the black bowl between the plate and the ramekin and place it on the plate",
|
| 7 |
+
"pick up the black bowl from table center and place it on the plate",
|
| 8 |
+
"pick up the black bowl in the top drawer of the wooden cabinet and place it on the plate",
|
| 9 |
+
"pick up the black bowl next to the cookie box and place it on the plate",
|
| 10 |
+
"pick up the black bowl next to the plate and place it on the plate",
|
| 11 |
+
"pick up the black bowl next to the ramekin and place it on the plate",
|
| 12 |
+
"pick up the black bowl on the cookie box and place it on the plate",
|
| 13 |
+
"pick up the black bowl on the ramekin and place it on the plate",
|
| 14 |
+
"pick up the black bowl on the stove and place it on the plate",
|
| 15 |
+
"pick up the black bowl on the wooden cabinet and place it on the plate",
|
| 16 |
+
"pick up the book and place it in the back compartment of the caddy",
|
| 17 |
+
"pick up the butter and place it in the basket",
|
| 18 |
+
"pick up the chocolate pudding and place it in the basket",
|
| 19 |
+
"pick up the cream cheese and place it in the basket",
|
| 20 |
+
"pick up the ketchup and place it in the basket",
|
| 21 |
+
"pick up the milk and place it in the basket",
|
| 22 |
+
"pick up the orange juice and place it in the basket",
|
| 23 |
+
"pick up the salad dressing and place it in the basket",
|
| 24 |
+
"pick up the tomato sauce and place it in the basket",
|
| 25 |
+
"push the plate to the front of the stove",
|
| 26 |
+
"put both moka pots on the stove",
|
| 27 |
+
"put both the alphabet soup and the cream cheese box in the basket",
|
| 28 |
+
"put both the alphabet soup and the tomato sauce in the basket",
|
| 29 |
+
"put both the cream cheese box and the butter in the basket",
|
| 30 |
+
"put the black bowl in the bottom drawer of the cabinet and close it",
|
| 31 |
+
"put the bowl on the plate",
|
| 32 |
+
"put the bowl on the stove",
|
| 33 |
+
"put the bowl on top of the cabinet",
|
| 34 |
+
"put the cream cheese in the bowl",
|
| 35 |
+
"put the white mug on the left plate and put the yellow and white mug on the right plate",
|
| 36 |
+
"put the white mug on the plate and put the chocolate pudding to the right of the plate",
|
| 37 |
+
"put the wine bottle on the rack",
|
| 38 |
+
"put the wine bottle on top of the cabinet",
|
| 39 |
+
"put the yellow and white mug in the microwave and close it",
|
| 40 |
+
"turn on the stove",
|
| 41 |
+
"turn on the stove and put the moka pot on it"
|
| 42 |
+
]
|
c_engine/export_from_safetensors.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Export Parvulus model weights, action scales, and canonical task embeddings from SafeTensors/JSON to binary format."""
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import json
|
| 6 |
+
import struct
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import numpy as np
|
| 10 |
+
|
| 11 |
+
_ROOT = Path(__file__).resolve().parent.parent
|
| 12 |
+
if str(_ROOT) not in sys.path:
|
| 13 |
+
sys.path.insert(0, str(_ROOT))
|
| 14 |
+
|
| 15 |
+
from safetensors.torch import load_file
|
| 16 |
+
|
| 17 |
+
MAGIC = 0x50415256 # "PARV"
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def export_binary_weights(safetensors_path: str, out_bin_path: str):
|
| 21 |
+
"""Exports model weights from SafeTensors into Parvulus pure C binary format."""
|
| 22 |
+
st_path = Path(safetensors_path)
|
| 23 |
+
if not st_path.is_absolute():
|
| 24 |
+
st_path = _ROOT / safetensors_path
|
| 25 |
+
|
| 26 |
+
out_p = Path(out_bin_path)
|
| 27 |
+
if not out_p.is_absolute():
|
| 28 |
+
out_p = _ROOT / out_bin_path
|
| 29 |
+
out_p.parent.mkdir(parents=True, exist_ok=True)
|
| 30 |
+
|
| 31 |
+
print(f"[Export] Loading weights from {st_path.name}...")
|
| 32 |
+
st_weights = load_file(str(st_path))
|
| 33 |
+
|
| 34 |
+
tensors_to_save = {}
|
| 35 |
+
for k, v in st_weights.items():
|
| 36 |
+
clean_k = k.replace("module.", "").replace("_orig_mod.", "")
|
| 37 |
+
if any(clean_k.startswith(p) for p in ["encoder.", "action_t_mlp.", "action_decoder."]):
|
| 38 |
+
tensors_to_save[clean_k] = v.float().contiguous().numpy()
|
| 39 |
+
|
| 40 |
+
print(f"[Export] Serializing {len(tensors_to_save)} tensors to {out_p.name}...")
|
| 41 |
+
|
| 42 |
+
header_magic = MAGIC
|
| 43 |
+
header_version = 1
|
| 44 |
+
num_tensors = len(tensors_to_save)
|
| 45 |
+
|
| 46 |
+
dir_bytes = bytearray()
|
| 47 |
+
data_bytes = bytearray()
|
| 48 |
+
|
| 49 |
+
current_offset = 0
|
| 50 |
+
for name, arr in tensors_to_save.items():
|
| 51 |
+
name_encoded = name.encode("utf-8")
|
| 52 |
+
name_len = len(name_encoded)
|
| 53 |
+
dims = list(arr.shape)
|
| 54 |
+
num_dims = len(dims)
|
| 55 |
+
padded_dims = dims + [0] * (4 - num_dims)
|
| 56 |
+
size_bytes = arr.nbytes
|
| 57 |
+
|
| 58 |
+
data_bytes.extend(arr.tobytes())
|
| 59 |
+
|
| 60 |
+
# Metadata format: name_len (H), name (s), num_dims (H), dims (4I), offset (Q), size (Q)
|
| 61 |
+
dir_bytes.extend(struct.pack("<H", name_len))
|
| 62 |
+
dir_bytes.extend(name_encoded)
|
| 63 |
+
dir_bytes.extend(struct.pack("<H4IQQ", num_dims, *padded_dims, current_offset, size_bytes))
|
| 64 |
+
|
| 65 |
+
current_offset += size_bytes
|
| 66 |
+
|
| 67 |
+
header_size = 16 + len(dir_bytes)
|
| 68 |
+
with open(out_p, "wb") as f:
|
| 69 |
+
f.write(struct.pack("<IIII", header_magic, header_version, num_tensors, header_size))
|
| 70 |
+
f.write(dir_bytes)
|
| 71 |
+
f.write(data_bytes)
|
| 72 |
+
|
| 73 |
+
total_mb = (len(dir_bytes) + len(data_bytes) + 16) / (1024 * 1024)
|
| 74 |
+
print(f"[Export] Successfully exported {total_mb:.2f} MB binary model to {out_p}")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def export_action_scale(json_path: str = "action_scale.json", out_bin_path: str = "c_engine/action_scale.bin"):
|
| 78 |
+
"""Exports 7-DoF min/max action normalizer bounds from JSON to 56-byte binary."""
|
| 79 |
+
in_p = Path(json_path)
|
| 80 |
+
if not in_p.is_absolute():
|
| 81 |
+
in_p = _ROOT / json_path
|
| 82 |
+
|
| 83 |
+
out_p = Path(out_bin_path)
|
| 84 |
+
if not out_p.is_absolute():
|
| 85 |
+
out_p = _ROOT / out_bin_path
|
| 86 |
+
out_p.parent.mkdir(parents=True, exist_ok=True)
|
| 87 |
+
|
| 88 |
+
with open(in_p, "r") as f:
|
| 89 |
+
data = json.load(f)
|
| 90 |
+
|
| 91 |
+
min_vals = np.array(data["min"], dtype=np.float32)
|
| 92 |
+
max_vals = np.array(data["max"], dtype=np.float32)
|
| 93 |
+
assert len(min_vals) == 7 and len(max_vals) == 7
|
| 94 |
+
|
| 95 |
+
packed = struct.pack("<14f", *min_vals, *max_vals)
|
| 96 |
+
with open(out_p, "wb") as f:
|
| 97 |
+
f.write(packed)
|
| 98 |
+
print(f"[Export] Exported action scale binary to {out_p} ({len(packed)} bytes)")
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def export_task_embeddings(
|
| 102 |
+
tasks_path: str = "c_engine/canonical_tasks.json",
|
| 103 |
+
safetensors_path: str = "parvulus.safetensors",
|
| 104 |
+
out_bin_path: str = "c_engine/task_embeddings.bin",
|
| 105 |
+
):
|
| 106 |
+
"""Encodes canonical benchmark tasks into pre-cached binary lookup table."""
|
| 107 |
+
import torch
|
| 108 |
+
from text_encoder import ParvulusTextEncoder
|
| 109 |
+
|
| 110 |
+
t_p = Path(tasks_path)
|
| 111 |
+
if not t_p.is_absolute():
|
| 112 |
+
t_p = _ROOT / tasks_path
|
| 113 |
+
|
| 114 |
+
st_p = Path(safetensors_path)
|
| 115 |
+
if not st_p.is_absolute():
|
| 116 |
+
st_p = _ROOT / safetensors_path
|
| 117 |
+
|
| 118 |
+
out_p = Path(out_bin_path)
|
| 119 |
+
if not out_p.is_absolute():
|
| 120 |
+
out_p = _ROOT / out_bin_path
|
| 121 |
+
out_p.parent.mkdir(parents=True, exist_ok=True)
|
| 122 |
+
|
| 123 |
+
with open(t_p, "r") as f:
|
| 124 |
+
tasks = json.load(f)
|
| 125 |
+
|
| 126 |
+
print(f"[Export] Encoding {len(tasks)} canonical tasks using ParvulusTextEncoder...")
|
| 127 |
+
te = ParvulusTextEncoder().cpu().eval()
|
| 128 |
+
st_weights = load_file(str(st_p))
|
| 129 |
+
proj_dict = {
|
| 130 |
+
"0.weight": st_weights["text_encoder.proj.0.weight"],
|
| 131 |
+
"0.bias": st_weights["text_encoder.proj.0.bias"],
|
| 132 |
+
"2.weight": st_weights["text_encoder.proj.2.weight"],
|
| 133 |
+
"2.bias": st_weights["text_encoder.proj.2.bias"],
|
| 134 |
+
}
|
| 135 |
+
te.proj.load_state_dict(proj_dict)
|
| 136 |
+
|
| 137 |
+
with open(out_p, "wb") as f:
|
| 138 |
+
f.write(struct.pack("<I", len(tasks)))
|
| 139 |
+
for t in tasks:
|
| 140 |
+
t_bytes = t.encode("utf-8")
|
| 141 |
+
f.write(struct.pack("<H", len(t_bytes)))
|
| 142 |
+
f.write(t_bytes)
|
| 143 |
+
with torch.no_grad():
|
| 144 |
+
emb = te([t]).numpy()[0].astype(np.float32)
|
| 145 |
+
f.write(emb.tobytes())
|
| 146 |
+
|
| 147 |
+
print(f"[Export] Successfully exported {len(tasks)} embeddings to {out_p} ({out_p.stat().st_size} bytes)")
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def main():
|
| 151 |
+
parser = argparse.ArgumentParser(description="Export Parvulus model weights and tables to binary format.")
|
| 152 |
+
parser.add_argument("--weights", type=str, default="parvulus.safetensors", help="Path to input SafeTensors weights")
|
| 153 |
+
parser.add_argument("--out_weights", type=str, default="c_engine/parvulus_model_weights.bin")
|
| 154 |
+
parser.add_argument("--scale_json", type=str, default="action_scale.json")
|
| 155 |
+
parser.add_argument("--out_scale", type=str, default="c_engine/action_scale.bin")
|
| 156 |
+
parser.add_argument("--tasks_json", type=str, default="c_engine/canonical_tasks.json")
|
| 157 |
+
parser.add_argument("--out_embeddings", type=str, default="c_engine/task_embeddings.bin")
|
| 158 |
+
parser.add_argument("--all", action="store_true", default=True, help="Export weights, scale, and embeddings")
|
| 159 |
+
|
| 160 |
+
args = parser.parse_args()
|
| 161 |
+
|
| 162 |
+
export_binary_weights(args.weights, args.out_weights)
|
| 163 |
+
export_action_scale(args.scale_json, args.out_scale)
|
| 164 |
+
if Path(_ROOT / args.tasks_json).exists():
|
| 165 |
+
export_task_embeddings(args.tasks_json, args.weights, args.out_embeddings)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
if __name__ == "__main__":
|
| 169 |
+
main()
|
c_engine/export_text_encoder_onnx.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Export Parvulus text encoder (T5-small + projection head) to standalone ONNX format.
|
| 3 |
+
|
| 4 |
+
Enables zero-PyTorch dynamic text encoding and perturbation handling via ONNX Runtime.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
import argparse
|
| 8 |
+
import sys
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
_ROOT = Path(__file__).resolve().parent.parent
|
| 12 |
+
if str(_ROOT) not in sys.path:
|
| 13 |
+
sys.path.insert(0, str(_ROOT))
|
| 14 |
+
|
| 15 |
+
import torch
|
| 16 |
+
import torch.nn as nn
|
| 17 |
+
from safetensors.torch import load_file
|
| 18 |
+
from text_encoder import ParvulusTextEncoder
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class ParvulusTextEncoderONNX(nn.Module):
|
| 22 |
+
"""Wraps T5EncoderModel + mean pooling + trained 2-layer projection head."""
|
| 23 |
+
|
| 24 |
+
def __init__(self, text_encoder: ParvulusTextEncoder):
|
| 25 |
+
super().__init__()
|
| 26 |
+
self.t5 = text_encoder.t5
|
| 27 |
+
self.proj = text_encoder.proj
|
| 28 |
+
|
| 29 |
+
def forward(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
|
| 30 |
+
out = self.t5(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
|
| 31 |
+
mask = attention_mask.unsqueeze(-1).to(out.dtype)
|
| 32 |
+
summed = (out * mask).sum(dim=1)
|
| 33 |
+
count = mask.sum(dim=1).clamp(min=1e-6)
|
| 34 |
+
pooled = summed / count
|
| 35 |
+
return self.proj(pooled)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def export_onnx(
|
| 39 |
+
weights_path: str = "parvulus.safetensors",
|
| 40 |
+
output_path: str = "c_engine/parvulus_text_encoder.onnx",
|
| 41 |
+
model_name: str = "google-t5/t5-small",
|
| 42 |
+
revision: str = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4",
|
| 43 |
+
opset_version: int = 14,
|
| 44 |
+
verify: bool = True,
|
| 45 |
+
):
|
| 46 |
+
"""Converts the Parvulus text encoder to standalone ONNX format."""
|
| 47 |
+
weights_p = Path(weights_path)
|
| 48 |
+
if not weights_p.is_absolute():
|
| 49 |
+
weights_p = _ROOT / weights_path
|
| 50 |
+
|
| 51 |
+
if not weights_p.exists():
|
| 52 |
+
fallback_p = _ROOT / "model.safetensors"
|
| 53 |
+
if fallback_p.exists():
|
| 54 |
+
weights_p = fallback_p
|
| 55 |
+
else:
|
| 56 |
+
raise FileNotFoundError(f"SafeTensors weights not found at {weights_p} or {fallback_p}")
|
| 57 |
+
|
| 58 |
+
out_p = Path(output_path)
|
| 59 |
+
if not out_p.is_absolute():
|
| 60 |
+
out_p = _ROOT / output_path
|
| 61 |
+
out_p.parent.mkdir(parents=True, exist_ok=True)
|
| 62 |
+
|
| 63 |
+
print(f"[Converter] Loading text encoder base ({model_name} @ {revision[:8]})...")
|
| 64 |
+
te = ParvulusTextEncoder(cond_dim=256, model_name=model_name, revision=revision).cpu().eval()
|
| 65 |
+
|
| 66 |
+
print(f"[Converter] Loading projection head weights from {weights_p.name}...")
|
| 67 |
+
weights = load_file(str(weights_p))
|
| 68 |
+
proj_dict = {
|
| 69 |
+
"0.weight": weights["text_encoder.proj.0.weight"],
|
| 70 |
+
"0.bias": weights["text_encoder.proj.0.bias"],
|
| 71 |
+
"2.weight": weights["text_encoder.proj.2.weight"],
|
| 72 |
+
"2.bias": weights["text_encoder.proj.2.bias"],
|
| 73 |
+
}
|
| 74 |
+
te.proj.load_state_dict(proj_dict)
|
| 75 |
+
|
| 76 |
+
model_onnx = ParvulusTextEncoderONNX(te).eval()
|
| 77 |
+
|
| 78 |
+
sample_prompt = "pick up the black bowl between the plate and the ramekin and place it on the plate"
|
| 79 |
+
tokens = te.tokenizer([sample_prompt], padding=True, truncation=True, return_tensors="pt")
|
| 80 |
+
|
| 81 |
+
print(f"[Converter] Exporting ONNX model to {out_p} (opset {opset_version})...")
|
| 82 |
+
torch.onnx.export(
|
| 83 |
+
model_onnx,
|
| 84 |
+
(tokens["input_ids"], tokens["attention_mask"]),
|
| 85 |
+
str(out_p),
|
| 86 |
+
input_names=["input_ids", "attention_mask"],
|
| 87 |
+
output_names=["text_emb"],
|
| 88 |
+
dynamic_axes={
|
| 89 |
+
"input_ids": {0: "batch", 1: "seq_len"},
|
| 90 |
+
"attention_mask": {0: "batch", 1: "seq_len"},
|
| 91 |
+
"text_emb": {0: "batch"},
|
| 92 |
+
},
|
| 93 |
+
opset_version=opset_version,
|
| 94 |
+
)
|
| 95 |
+
print(f"[Converter] Successfully exported ONNX model ({out_p.stat().st_size / (1024*1024):.1f} MB).")
|
| 96 |
+
|
| 97 |
+
if verify:
|
| 98 |
+
try:
|
| 99 |
+
import onnxruntime as ort
|
| 100 |
+
|
| 101 |
+
sess = ort.InferenceSession(str(out_p))
|
| 102 |
+
ort_out = sess.run(
|
| 103 |
+
None,
|
| 104 |
+
{
|
| 105 |
+
"input_ids": tokens["input_ids"].numpy(),
|
| 106 |
+
"attention_mask": tokens["attention_mask"].numpy(),
|
| 107 |
+
},
|
| 108 |
+
)[0]
|
| 109 |
+
ref_emb = te([sample_prompt]).detach().numpy()
|
| 110 |
+
max_diff = float(abs(ref_emb - ort_out).max())
|
| 111 |
+
print(f"[Converter] Numerical parity verified! Max absolute diff vs PyTorch: {max_diff:.3e}")
|
| 112 |
+
except ImportError:
|
| 113 |
+
print("[Converter] onnxruntime not installed; skipping verification check.")
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def main():
|
| 117 |
+
parser = argparse.ArgumentParser(
|
| 118 |
+
description="Convert Parvulus text encoder to standalone ONNX format for zero-PyTorch dynamic text encoding."
|
| 119 |
+
)
|
| 120 |
+
parser.add_argument(
|
| 121 |
+
"--weights",
|
| 122 |
+
type=str,
|
| 123 |
+
default="parvulus.safetensors",
|
| 124 |
+
help="Path to parvulus.safetensors or model.safetensors",
|
| 125 |
+
)
|
| 126 |
+
parser.add_argument(
|
| 127 |
+
"--output",
|
| 128 |
+
type=str,
|
| 129 |
+
default="c_engine/parvulus_text_encoder.onnx",
|
| 130 |
+
help="Path to save output ONNX model",
|
| 131 |
+
)
|
| 132 |
+
parser.add_argument(
|
| 133 |
+
"--model_name",
|
| 134 |
+
type=str,
|
| 135 |
+
default="google-t5/t5-small",
|
| 136 |
+
help="Base T5 model name on Hugging Face Hub",
|
| 137 |
+
)
|
| 138 |
+
parser.add_argument(
|
| 139 |
+
"--revision",
|
| 140 |
+
type=str,
|
| 141 |
+
default="df1b051c49625cf57a3d0d8d3863ed4d13564fe4",
|
| 142 |
+
help="Pinned commit revision on Hugging Face Hub",
|
| 143 |
+
)
|
| 144 |
+
parser.add_argument(
|
| 145 |
+
"--opset",
|
| 146 |
+
type=int,
|
| 147 |
+
default=14,
|
| 148 |
+
help="ONNX opset version (default: 14)",
|
| 149 |
+
)
|
| 150 |
+
parser.add_argument(
|
| 151 |
+
"--no-verify",
|
| 152 |
+
action="store_true",
|
| 153 |
+
help="Skip numerical verification with onnxruntime",
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
args = parser.parse_args()
|
| 157 |
+
export_onnx(
|
| 158 |
+
weights_path=args.weights,
|
| 159 |
+
output_path=args.output,
|
| 160 |
+
model_name=args.model_name,
|
| 161 |
+
revision=args.revision,
|
| 162 |
+
opset_version=args.opset,
|
| 163 |
+
verify=not args.no_verify,
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
if __name__ == "__main__":
|
| 168 |
+
main()
|
c_engine/infer_c.py
ADDED
|
@@ -0,0 +1,443 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""C++ and CUDA inference backends for Parvulus."""
|
| 2 |
+
|
| 3 |
+
import ctypes
|
| 4 |
+
import os
|
| 5 |
+
import time
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Dict, List, Optional, Sequence, Tuple, Union
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
|
| 11 |
+
C_ENGINE_DIR = Path(__file__).resolve().parent
|
| 12 |
+
LIB_PATH = C_ENGINE_DIR / "libparvulus.so"
|
| 13 |
+
|
| 14 |
+
_lib = None
|
| 15 |
+
if LIB_PATH.exists():
|
| 16 |
+
try:
|
| 17 |
+
_lib = ctypes.CDLL(str(LIB_PATH))
|
| 18 |
+
_lib.parvulus_create.argtypes = [ctypes.c_int]
|
| 19 |
+
_lib.parvulus_create.restype = ctypes.c_void_p
|
| 20 |
+
|
| 21 |
+
_lib.parvulus_load.argtypes = [
|
| 22 |
+
ctypes.c_void_p,
|
| 23 |
+
ctypes.c_char_p,
|
| 24 |
+
ctypes.c_char_p,
|
| 25 |
+
ctypes.c_char_p,
|
| 26 |
+
]
|
| 27 |
+
_lib.parvulus_load.restype = ctypes.c_bool
|
| 28 |
+
|
| 29 |
+
_lib.parvulus_set_task_vector.argtypes = [
|
| 30 |
+
ctypes.c_void_p,
|
| 31 |
+
ctypes.POINTER(ctypes.c_float),
|
| 32 |
+
]
|
| 33 |
+
_lib.parvulus_set_task_vector.restype = ctypes.c_bool
|
| 34 |
+
|
| 35 |
+
_lib.parvulus_set_task_string.argtypes = [
|
| 36 |
+
ctypes.c_void_p,
|
| 37 |
+
ctypes.c_char_p,
|
| 38 |
+
]
|
| 39 |
+
_lib.parvulus_set_task_string.restype = ctypes.c_bool
|
| 40 |
+
|
| 41 |
+
_lib.parvulus_predict_chunk.argtypes = [
|
| 42 |
+
ctypes.c_void_p,
|
| 43 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 44 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 45 |
+
ctypes.c_char_p,
|
| 46 |
+
ctypes.POINTER(ctypes.c_float),
|
| 47 |
+
ctypes.c_int,
|
| 48 |
+
ctypes.POINTER(ctypes.c_float),
|
| 49 |
+
]
|
| 50 |
+
_lib.parvulus_predict_chunk.restype = ctypes.c_bool
|
| 51 |
+
|
| 52 |
+
_lib.parvulus_predict_chunk_with_vector.argtypes = [
|
| 53 |
+
ctypes.c_void_p,
|
| 54 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 55 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 56 |
+
ctypes.POINTER(ctypes.c_float),
|
| 57 |
+
ctypes.POINTER(ctypes.c_float),
|
| 58 |
+
ctypes.c_int,
|
| 59 |
+
ctypes.POINTER(ctypes.c_float),
|
| 60 |
+
]
|
| 61 |
+
_lib.parvulus_predict_chunk_with_vector.restype = ctypes.c_bool
|
| 62 |
+
|
| 63 |
+
_lib.parvulus_predict_chunk_with_emb.argtypes = [
|
| 64 |
+
ctypes.c_void_p,
|
| 65 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 66 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 67 |
+
ctypes.POINTER(ctypes.c_float),
|
| 68 |
+
ctypes.POINTER(ctypes.c_float),
|
| 69 |
+
ctypes.c_int,
|
| 70 |
+
ctypes.POINTER(ctypes.c_float),
|
| 71 |
+
]
|
| 72 |
+
_lib.parvulus_predict_chunk_with_emb.restype = ctypes.c_bool
|
| 73 |
+
|
| 74 |
+
_lib.parvulus_register_task_embedding.argtypes = [
|
| 75 |
+
ctypes.c_void_p,
|
| 76 |
+
ctypes.c_char_p,
|
| 77 |
+
ctypes.POINTER(ctypes.c_float),
|
| 78 |
+
]
|
| 79 |
+
_lib.parvulus_register_task_embedding.restype = ctypes.c_bool
|
| 80 |
+
|
| 81 |
+
_lib.parvulus_destroy.argtypes = [ctypes.c_void_p]
|
| 82 |
+
_lib.parvulus_destroy.restype = None
|
| 83 |
+
except Exception as e:
|
| 84 |
+
print(f"[Warning] Could not load libparvulus.so: {e}")
|
| 85 |
+
_lib = None
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
class ParvulusEngine:
|
| 89 |
+
"""Native Parvulus CUDA/cuBLAS runtime engine (Zero LibTorch, Zero PyTorch)."""
|
| 90 |
+
|
| 91 |
+
def __init__(
|
| 92 |
+
self,
|
| 93 |
+
device_id: int = 0,
|
| 94 |
+
weights_path: Optional[Union[str, Path]] = None,
|
| 95 |
+
embeddings_path: Optional[Union[str, Path]] = None,
|
| 96 |
+
scale_path: Optional[Union[str, Path]] = None,
|
| 97 |
+
chunk_size: int = 8,
|
| 98 |
+
replan_stride: int = 8,
|
| 99 |
+
num_ode_steps: int = 10,
|
| 100 |
+
):
|
| 101 |
+
if _lib is None:
|
| 102 |
+
raise RuntimeError(f"libparvulus.so not loaded. Check: {LIB_PATH}")
|
| 103 |
+
|
| 104 |
+
self.device_id = device_id
|
| 105 |
+
self.chunk_size = chunk_size
|
| 106 |
+
self.replan_stride = replan_stride
|
| 107 |
+
self.num_ode_steps = num_ode_steps
|
| 108 |
+
|
| 109 |
+
if weights_path is None:
|
| 110 |
+
weights_path = C_ENGINE_DIR / "parvulus_model_weights.bin"
|
| 111 |
+
if embeddings_path is None:
|
| 112 |
+
embeddings_path = C_ENGINE_DIR / "task_embeddings.bin"
|
| 113 |
+
if scale_path is None:
|
| 114 |
+
scale_path = C_ENGINE_DIR / "action_scale.bin"
|
| 115 |
+
|
| 116 |
+
weights_path = Path(weights_path)
|
| 117 |
+
embeddings_path = Path(embeddings_path)
|
| 118 |
+
scale_path = Path(scale_path)
|
| 119 |
+
|
| 120 |
+
if not weights_path.exists():
|
| 121 |
+
print(f"[ParvulusEngine] Generating {weights_path.name} from SafeTensors...")
|
| 122 |
+
import sys
|
| 123 |
+
if str(C_ENGINE_DIR) not in sys.path:
|
| 124 |
+
sys.path.insert(0, str(C_ENGINE_DIR))
|
| 125 |
+
from export_from_safetensors import export_binary_weights, export_action_scale, export_task_embeddings
|
| 126 |
+
|
| 127 |
+
st_path = C_ENGINE_DIR.parent / "parvulus.safetensors"
|
| 128 |
+
if not st_path.exists():
|
| 129 |
+
st_path = C_ENGINE_DIR.parent / "model.safetensors"
|
| 130 |
+
export_binary_weights(str(st_path), str(weights_path))
|
| 131 |
+
if not scale_path.exists():
|
| 132 |
+
export_action_scale(str(C_ENGINE_DIR.parent / "action_scale.json"), str(scale_path))
|
| 133 |
+
if not embeddings_path.exists() and (C_ENGINE_DIR / "canonical_tasks.json").exists():
|
| 134 |
+
export_task_embeddings(str(C_ENGINE_DIR / "canonical_tasks.json"), str(st_path), str(embeddings_path))
|
| 135 |
+
|
| 136 |
+
self.handle = _lib.parvulus_create(self.device_id)
|
| 137 |
+
if not self.handle:
|
| 138 |
+
raise RuntimeError(f"Failed to initialize Parvulus engine on device {self.device_id}")
|
| 139 |
+
|
| 140 |
+
ok = _lib.parvulus_load(
|
| 141 |
+
self.handle,
|
| 142 |
+
str(weights_path).encode("utf-8"),
|
| 143 |
+
str(embeddings_path).encode("utf-8"),
|
| 144 |
+
str(scale_path).encode("utf-8"),
|
| 145 |
+
)
|
| 146 |
+
if not ok:
|
| 147 |
+
_lib.parvulus_destroy(self.handle)
|
| 148 |
+
self.handle = None
|
| 149 |
+
raise RuntimeError("Failed to load Parvulus engine weights!")
|
| 150 |
+
|
| 151 |
+
self.current_task_desc: Optional[str] = None
|
| 152 |
+
self.action_queue = []
|
| 153 |
+
self.stride_counter: int = 0
|
| 154 |
+
self._actions_buffer = np.zeros(56, dtype=np.float32)
|
| 155 |
+
|
| 156 |
+
self._onnx_session = None
|
| 157 |
+
self._tokenizer = None
|
| 158 |
+
self._torch_text_encoder = None
|
| 159 |
+
|
| 160 |
+
def __del__(self):
|
| 161 |
+
if hasattr(self, "handle") and self.handle is not None and _lib is not None:
|
| 162 |
+
_lib.parvulus_destroy(self.handle)
|
| 163 |
+
self.handle = None
|
| 164 |
+
|
| 165 |
+
def set_task_vector(self, task_vector: np.ndarray) -> bool:
|
| 166 |
+
"""Sets the active 256-dim task conditioning vector on the GPU."""
|
| 167 |
+
vec_c = np.ascontiguousarray(task_vector.flatten()[:256], dtype=np.float32)
|
| 168 |
+
vec_ptr = vec_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 169 |
+
return bool(_lib.parvulus_set_task_vector(self.handle, vec_ptr))
|
| 170 |
+
|
| 171 |
+
def encode_text(self, text: str) -> np.ndarray:
|
| 172 |
+
"""Encodes arbitrary natural language prompts or perturbations into 256-dim vector."""
|
| 173 |
+
onnx_path = C_ENGINE_DIR / "parvulus_text_encoder.onnx"
|
| 174 |
+
if not onnx_path.exists() and os.environ.get("PARVULUS_HF_REPO"):
|
| 175 |
+
try:
|
| 176 |
+
from huggingface_hub import hf_hub_download
|
| 177 |
+
hf_hub_download(
|
| 178 |
+
repo_id=os.environ["PARVULUS_HF_REPO"],
|
| 179 |
+
filename="parvulus_text_encoder.onnx",
|
| 180 |
+
subfolder="c_engine",
|
| 181 |
+
local_dir=str(C_ENGINE_DIR.parent),
|
| 182 |
+
)
|
| 183 |
+
except Exception:
|
| 184 |
+
pass
|
| 185 |
+
|
| 186 |
+
if onnx_path.exists():
|
| 187 |
+
try:
|
| 188 |
+
import onnxruntime as ort
|
| 189 |
+
from transformers import AutoTokenizer
|
| 190 |
+
|
| 191 |
+
if self._onnx_session is None:
|
| 192 |
+
available = ort.get_available_providers()
|
| 193 |
+
providers = [p for p in ["CUDAExecutionProvider", "CPUExecutionProvider"] if p in available]
|
| 194 |
+
self._onnx_session = ort.InferenceSession(str(onnx_path), providers=providers)
|
| 195 |
+
self._tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small")
|
| 196 |
+
|
| 197 |
+
tokens = self._tokenizer([text], padding=True, truncation=True, return_tensors="np")
|
| 198 |
+
ort_out = self._onnx_session.run(
|
| 199 |
+
None,
|
| 200 |
+
{
|
| 201 |
+
"input_ids": tokens["input_ids"].astype(np.int64),
|
| 202 |
+
"attention_mask": tokens["attention_mask"].astype(np.int64),
|
| 203 |
+
},
|
| 204 |
+
)[0]
|
| 205 |
+
return ort_out[0].astype(np.float32)
|
| 206 |
+
except Exception:
|
| 207 |
+
pass
|
| 208 |
+
|
| 209 |
+
try:
|
| 210 |
+
from text_encoder import ParvulusTextEncoder
|
| 211 |
+
from safetensors.torch import load_file
|
| 212 |
+
|
| 213 |
+
if self._torch_text_encoder is None:
|
| 214 |
+
te = ParvulusTextEncoder().cpu().eval()
|
| 215 |
+
st_path = C_ENGINE_DIR.parent / "parvulus.safetensors"
|
| 216 |
+
if st_path.exists():
|
| 217 |
+
weights = load_file(str(st_path))
|
| 218 |
+
proj_dict = {
|
| 219 |
+
"0.weight": weights["text_encoder.proj.0.weight"],
|
| 220 |
+
"0.bias": weights["text_encoder.proj.0.bias"],
|
| 221 |
+
"2.weight": weights["text_encoder.proj.2.weight"],
|
| 222 |
+
"2.bias": weights["text_encoder.proj.2.bias"],
|
| 223 |
+
}
|
| 224 |
+
te.proj.load_state_dict(proj_dict)
|
| 225 |
+
self._torch_text_encoder = te
|
| 226 |
+
|
| 227 |
+
with torch.no_grad():
|
| 228 |
+
emb = self._torch_text_encoder([text]).cpu().numpy()[0]
|
| 229 |
+
return emb.astype(np.float32)
|
| 230 |
+
except Exception as e:
|
| 231 |
+
raise RuntimeError(
|
| 232 |
+
f"Failed to dynamically encode text prompt '{text}': {e}. "
|
| 233 |
+
"Ensure onnxruntime or transformers/torch is installed."
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
def reset(
|
| 237 |
+
self,
|
| 238 |
+
task_description: Optional[str] = None,
|
| 239 |
+
task_vector: Optional[np.ndarray] = None,
|
| 240 |
+
):
|
| 241 |
+
if task_vector is not None:
|
| 242 |
+
self.set_task_vector(task_vector)
|
| 243 |
+
self.current_task_desc = task_description or "<vector>"
|
| 244 |
+
elif task_description is not None:
|
| 245 |
+
self.current_task_desc = task_description
|
| 246 |
+
task_bytes = task_description.encode("utf-8")
|
| 247 |
+
if not _lib.parvulus_set_task_string(self.handle, task_bytes):
|
| 248 |
+
encoded_vec = self.encode_text(task_description)
|
| 249 |
+
self.set_task_vector(encoded_vec)
|
| 250 |
+
|
| 251 |
+
self.action_queue.clear()
|
| 252 |
+
self.stride_counter = 0
|
| 253 |
+
|
| 254 |
+
def predict_chunk(
|
| 255 |
+
self,
|
| 256 |
+
agentview_img: np.ndarray,
|
| 257 |
+
wrist_img: np.ndarray,
|
| 258 |
+
task_description: Optional[str] = None,
|
| 259 |
+
task_vector: Optional[np.ndarray] = None,
|
| 260 |
+
custom_z0: Optional[np.ndarray] = None,
|
| 261 |
+
) -> np.ndarray:
|
| 262 |
+
if task_vector is not None:
|
| 263 |
+
self.set_task_vector(task_vector)
|
| 264 |
+
elif task_description is not None:
|
| 265 |
+
self.reset(task_description=task_description)
|
| 266 |
+
|
| 267 |
+
agent_c = np.ascontiguousarray(agentview_img, dtype=np.uint8)
|
| 268 |
+
wrist_c = np.ascontiguousarray(wrist_img, dtype=np.uint8)
|
| 269 |
+
|
| 270 |
+
agent_ptr = agent_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
|
| 271 |
+
wrist_ptr = wrist_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
|
| 272 |
+
out_ptr = self._actions_buffer.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 273 |
+
|
| 274 |
+
z0_ptr = None
|
| 275 |
+
if custom_z0 is not None:
|
| 276 |
+
z0_c = np.ascontiguousarray(custom_z0, dtype=np.float32)
|
| 277 |
+
z0_ptr = z0_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 278 |
+
|
| 279 |
+
ok = _lib.parvulus_predict_chunk(
|
| 280 |
+
self.handle,
|
| 281 |
+
agent_ptr,
|
| 282 |
+
wrist_ptr,
|
| 283 |
+
None,
|
| 284 |
+
out_ptr,
|
| 285 |
+
self.num_ode_steps,
|
| 286 |
+
z0_ptr,
|
| 287 |
+
)
|
| 288 |
+
if not ok:
|
| 289 |
+
raise RuntimeError("Parvulus predict_chunk failed!")
|
| 290 |
+
|
| 291 |
+
return self._actions_buffer.reshape(8, 7).copy()
|
| 292 |
+
|
| 293 |
+
def predict_chunk_with_emb(
|
| 294 |
+
self,
|
| 295 |
+
agentview_img: np.ndarray,
|
| 296 |
+
wrist_img: np.ndarray,
|
| 297 |
+
text_embedding: np.ndarray,
|
| 298 |
+
custom_z0: Optional[np.ndarray] = None,
|
| 299 |
+
) -> np.ndarray:
|
| 300 |
+
"""Predicts action chunk using a directly provided 256-dim text embedding vector."""
|
| 301 |
+
return self.predict_chunk(
|
| 302 |
+
agentview_img,
|
| 303 |
+
wrist_img,
|
| 304 |
+
task_vector=text_embedding,
|
| 305 |
+
custom_z0=custom_z0,
|
| 306 |
+
)
|
| 307 |
+
|
| 308 |
+
def register_task_embedding(self, task_description: str, text_embedding: np.ndarray) -> bool:
|
| 309 |
+
"""Dynamically registers or updates a task embedding vector in engine memory."""
|
| 310 |
+
task_bytes = task_description.encode("utf-8")
|
| 311 |
+
emb_c = np.ascontiguousarray(text_embedding.flatten()[:256], dtype=np.float32)
|
| 312 |
+
emb_ptr = emb_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 313 |
+
return bool(_lib.parvulus_register_task_embedding(self.handle, task_bytes, emb_ptr))
|
| 314 |
+
|
| 315 |
+
def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray:
|
| 316 |
+
if not self.action_queue or self.stride_counter >= self.replan_stride:
|
| 317 |
+
chunk = self.predict_chunk(agentview_img, wrist_img)
|
| 318 |
+
self.action_queue = list(chunk[: self.replan_stride])
|
| 319 |
+
self.stride_counter = 0
|
| 320 |
+
|
| 321 |
+
action = self.action_queue.pop(0)
|
| 322 |
+
self.stride_counter += 1
|
| 323 |
+
return action
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
# Aliases for backward compatibility
|
| 327 |
+
ParvulusPureCUDAEngine = ParvulusEngine
|
| 328 |
+
ParvulusCUDAEngine = ParvulusEngine
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
class ParvulusCUDAGraphEngine:
|
| 332 |
+
"""CUDA Graph replay engine for PyTorch models.
|
| 333 |
+
|
| 334 |
+
Pre-allocates static GPU buffers, captures the complete ODE Transformer loop into
|
| 335 |
+
a CUDA execution graph, and replays it with zero host kernel launch overhead.
|
| 336 |
+
"""
|
| 337 |
+
|
| 338 |
+
def __init__(
|
| 339 |
+
self,
|
| 340 |
+
device_id: int = 0,
|
| 341 |
+
model_path: Optional[Union[str, Path]] = None,
|
| 342 |
+
embeddings_path: Optional[Union[str, Path]] = None,
|
| 343 |
+
scale_path: Optional[Union[str, Path]] = None,
|
| 344 |
+
chunk_size: int = 8,
|
| 345 |
+
replan_stride: int = 8,
|
| 346 |
+
num_ode_steps: int = 10,
|
| 347 |
+
):
|
| 348 |
+
self.device = torch.device(f"cuda:{device_id}")
|
| 349 |
+
self.chunk_size = chunk_size
|
| 350 |
+
self.replan_stride = replan_stride
|
| 351 |
+
self.num_ode_steps = num_ode_steps
|
| 352 |
+
|
| 353 |
+
if model_path is None:
|
| 354 |
+
model_path = C_ENGINE_DIR / "parvulus_traced_model.pt"
|
| 355 |
+
|
| 356 |
+
if embeddings_path is None:
|
| 357 |
+
embeddings_path = C_ENGINE_DIR / "task_embeddings.bin"
|
| 358 |
+
if scale_path is None:
|
| 359 |
+
scale_path = C_ENGINE_DIR / "action_scale.bin"
|
| 360 |
+
|
| 361 |
+
import struct
|
| 362 |
+
with open(scale_path, "rb") as f:
|
| 363 |
+
min_vals = struct.unpack("<7f", f.read(28))
|
| 364 |
+
max_vals = struct.unpack("<7f", f.read(28))
|
| 365 |
+
self.action_min = torch.tensor(min_vals, device=self.device, dtype=torch.float32).view(1, 1, 7)
|
| 366 |
+
self.action_max = torch.tensor(max_vals, device=self.device, dtype=torch.float32).view(1, 1, 7)
|
| 367 |
+
|
| 368 |
+
self.task_embeddings: Dict[str, torch.Tensor] = {}
|
| 369 |
+
with open(embeddings_path, "rb") as f:
|
| 370 |
+
num_tasks = struct.unpack("<I", f.read(4))[0]
|
| 371 |
+
for _ in range(num_tasks):
|
| 372 |
+
s_len = struct.unpack("<H", f.read(2))[0]
|
| 373 |
+
task_str = f.read(s_len).decode("utf-8")
|
| 374 |
+
emb = struct.unpack("<256f", f.read(1024))
|
| 375 |
+
self.task_embeddings[task_str] = torch.tensor(emb, device=self.device, dtype=torch.float32).view(1, 256)
|
| 376 |
+
|
| 377 |
+
if Path(model_path).exists():
|
| 378 |
+
self.model = torch.jit.load(str(model_path), map_location=self.device).eval()
|
| 379 |
+
|
| 380 |
+
self.static_obs = torch.zeros(1, 6, 256, 256, device=self.device, dtype=torch.float32)
|
| 381 |
+
self.static_text = torch.zeros(1, 256, device=self.device, dtype=torch.float32)
|
| 382 |
+
self.static_z0 = torch.zeros(1, 8, 7, device=self.device, dtype=torch.float32)
|
| 383 |
+
self.static_out = torch.zeros(1, 8, 7, device=self.device, dtype=torch.float32)
|
| 384 |
+
|
| 385 |
+
s = torch.cuda.Stream(device=self.device)
|
| 386 |
+
s.wait_stream(torch.cuda.current_stream(device=self.device))
|
| 387 |
+
with torch.cuda.stream(s):
|
| 388 |
+
for _ in range(5):
|
| 389 |
+
_ = self.model(self.static_obs, self.static_text, self.static_z0)
|
| 390 |
+
torch.cuda.current_stream(device=self.device).wait_stream(s)
|
| 391 |
+
|
| 392 |
+
self.graph = torch.cuda.CUDAGraph()
|
| 393 |
+
with torch.cuda.graph(self.graph, stream=s):
|
| 394 |
+
self.static_out = self.model(self.static_obs, self.static_text, self.static_z0)
|
| 395 |
+
else:
|
| 396 |
+
self.model = None
|
| 397 |
+
self.graph = None
|
| 398 |
+
|
| 399 |
+
self.current_task_desc: Optional[str] = None
|
| 400 |
+
self.action_queue = []
|
| 401 |
+
self.stride_counter: int = 0
|
| 402 |
+
|
| 403 |
+
def reset(self, task_description: Optional[str] = None):
|
| 404 |
+
if task_description is not None:
|
| 405 |
+
self.current_task_desc = task_description
|
| 406 |
+
if self.graph is not None and task_description in self.task_embeddings:
|
| 407 |
+
self.static_text.copy_(self.task_embeddings[task_description])
|
| 408 |
+
self.action_queue.clear()
|
| 409 |
+
self.stride_counter = 0
|
| 410 |
+
|
| 411 |
+
def predict_chunk(
|
| 412 |
+
self,
|
| 413 |
+
agentview_img: np.ndarray,
|
| 414 |
+
wrist_img: np.ndarray,
|
| 415 |
+
task_description: Optional[str] = None,
|
| 416 |
+
) -> np.ndarray:
|
| 417 |
+
if self.graph is None:
|
| 418 |
+
raise RuntimeError("ParvulusCUDAGraphEngine requires a valid TorchScript model.")
|
| 419 |
+
|
| 420 |
+
if task_description is not None:
|
| 421 |
+
self.reset(task_description)
|
| 422 |
+
|
| 423 |
+
combined_np = np.stack([agentview_img, wrist_img], axis=0)
|
| 424 |
+
combined_t = torch.from_numpy(combined_np).to(self.device, non_blocking=True).float()
|
| 425 |
+
combined_t = combined_t.flip(1, 2)
|
| 426 |
+
combined_t = (combined_t / 127.5) - 1.0
|
| 427 |
+
self.static_obs.copy_(combined_t.permute(0, 3, 1, 2).reshape(1, 6, 256, 256))
|
| 428 |
+
|
| 429 |
+
self.static_z0.normal_()
|
| 430 |
+
self.graph.replay()
|
| 431 |
+
|
| 432 |
+
actions = (self.static_out + 1.0) * 0.5 * (self.action_max - self.action_min) + self.action_min
|
| 433 |
+
return actions.squeeze(0).detach().cpu().numpy()
|
| 434 |
+
|
| 435 |
+
def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray:
|
| 436 |
+
if not self.action_queue or self.stride_counter >= self.replan_stride:
|
| 437 |
+
chunk = self.predict_chunk(agentview_img, wrist_img)
|
| 438 |
+
self.action_queue = list(chunk[: self.replan_stride])
|
| 439 |
+
self.stride_counter = 0
|
| 440 |
+
|
| 441 |
+
action = self.action_queue.pop(0)
|
| 442 |
+
self.stride_counter += 1
|
| 443 |
+
return action
|
c_engine/kernels.cuh
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
#include <cuda_runtime.h>
|
| 3 |
+
#include <cmath>
|
| 4 |
+
#include <cstdint>
|
| 5 |
+
|
| 6 |
+
#define WARP_SIZE 32
|
| 7 |
+
|
| 8 |
+
// -------------------------------------------------------------
|
| 9 |
+
// 1. Dual-POV Frame Preprocessing Kernel
|
| 10 |
+
// -------------------------------------------------------------
|
| 11 |
+
// Ingests HWC uint8 raw agentview & wristview frames (256x256x3)
|
| 12 |
+
// Performs Robosuite [::-1, ::-1] camera flip and [-1, 1] normalization
|
| 13 |
+
// Writes contiguous [6, 256, 256] float tensor directly to GPU memory
|
| 14 |
+
__global__ void preprocess_frames_kernel(
|
| 15 |
+
const uint8_t* __restrict__ agent_img,
|
| 16 |
+
const uint8_t* __restrict__ wrist_img,
|
| 17 |
+
float* __restrict__ out_obs,
|
| 18 |
+
int H, int W
|
| 19 |
+
) {
|
| 20 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 21 |
+
int total_pixels = H * W;
|
| 22 |
+
if (idx >= total_pixels) return;
|
| 23 |
+
|
| 24 |
+
int r = idx / W;
|
| 25 |
+
int c = idx % W;
|
| 26 |
+
|
| 27 |
+
// Robosuite [::-1, ::-1] flip
|
| 28 |
+
int flipped_r = H - 1 - r;
|
| 29 |
+
int flipped_c = W - 1 - c;
|
| 30 |
+
int in_idx = (flipped_r * W + flipped_c) * 3;
|
| 31 |
+
|
| 32 |
+
// Agent (Channels 0, 1, 2)
|
| 33 |
+
out_obs[0 * total_pixels + idx] = ((float)agent_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f;
|
| 34 |
+
out_obs[1 * total_pixels + idx] = ((float)agent_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f;
|
| 35 |
+
out_obs[2 * total_pixels + idx] = ((float)agent_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f;
|
| 36 |
+
|
| 37 |
+
// Wrist (Channels 3, 4, 5)
|
| 38 |
+
out_obs[3 * total_pixels + idx] = ((float)wrist_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f;
|
| 39 |
+
out_obs[4 * total_pixels + idx] = ((float)wrist_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f;
|
| 40 |
+
out_obs[5 * total_pixels + idx] = ((float)wrist_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f;
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
// -------------------------------------------------------------
|
| 44 |
+
// 2. LayerNorm Kernel (dim=256)
|
| 45 |
+
// -------------------------------------------------------------
|
| 46 |
+
__global__ void layernorm_256_kernel(
|
| 47 |
+
const float* __restrict__ in,
|
| 48 |
+
const float* __restrict__ gamma,
|
| 49 |
+
const float* __restrict__ beta,
|
| 50 |
+
float* __restrict__ out,
|
| 51 |
+
int num_rows,
|
| 52 |
+
float eps = 1e-5f
|
| 53 |
+
) {
|
| 54 |
+
int row = blockIdx.x;
|
| 55 |
+
if (row >= num_rows) return;
|
| 56 |
+
|
| 57 |
+
int tid = threadIdx.x;
|
| 58 |
+
const float* in_row = in + row * 256;
|
| 59 |
+
float* out_row = out + row * 256;
|
| 60 |
+
|
| 61 |
+
float val = (tid < 256) ? in_row[tid] : 0.0f;
|
| 62 |
+
|
| 63 |
+
// Compute mean
|
| 64 |
+
__shared__ float s_mean;
|
| 65 |
+
__shared__ float s_var;
|
| 66 |
+
|
| 67 |
+
// Warp-level reduction for 256 floats (8 warps of 32 threads)
|
| 68 |
+
float sum = val;
|
| 69 |
+
#pragma unroll
|
| 70 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 71 |
+
sum += __shfl_down_sync(0xffffffff, sum, offset);
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
__shared__ float warp_sums[8];
|
| 75 |
+
int lane = tid % 32;
|
| 76 |
+
int warp_id = tid / 32;
|
| 77 |
+
if (lane == 0) warp_sums[warp_id] = sum;
|
| 78 |
+
__syncthreads();
|
| 79 |
+
|
| 80 |
+
if (tid == 0) {
|
| 81 |
+
float total_sum = 0.0f;
|
| 82 |
+
for (int i = 0; i < 8; ++i) total_sum += warp_sums[i];
|
| 83 |
+
s_mean = total_sum / 256.0f;
|
| 84 |
+
}
|
| 85 |
+
__syncthreads();
|
| 86 |
+
|
| 87 |
+
float mean = s_mean;
|
| 88 |
+
float diff = val - mean;
|
| 89 |
+
float sq_diff = diff * diff;
|
| 90 |
+
|
| 91 |
+
#pragma unroll
|
| 92 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 93 |
+
sq_diff += __shfl_down_sync(0xffffffff, sq_diff, offset);
|
| 94 |
+
}
|
| 95 |
+
if (lane == 0) warp_sums[warp_id] = sq_diff;
|
| 96 |
+
__syncthreads();
|
| 97 |
+
|
| 98 |
+
if (tid == 0) {
|
| 99 |
+
float total_sq = 0.0f;
|
| 100 |
+
for (int i = 0; i < 8; ++i) total_sq += warp_sums[i];
|
| 101 |
+
s_var = total_sq / 256.0f;
|
| 102 |
+
}
|
| 103 |
+
__syncthreads();
|
| 104 |
+
|
| 105 |
+
float inv_std = rsqrtf(s_var + eps);
|
| 106 |
+
float g = (gamma != nullptr) ? gamma[tid] : 1.0f;
|
| 107 |
+
float b = (beta != nullptr) ? beta[tid] : 0.0f;
|
| 108 |
+
|
| 109 |
+
out_row[tid] = (diff * inv_std) * g + b;
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
// -------------------------------------------------------------
|
| 113 |
+
// 3. GELU Activation Kernel
|
| 114 |
+
// -------------------------------------------------------------
|
| 115 |
+
__global__ void gelu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) {
|
| 116 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 117 |
+
if (idx >= N) return;
|
| 118 |
+
float x = in[idx];
|
| 119 |
+
// Exact PyTorch GELU: x * 0.5 * (1 + erf(x / sqrt(2)))
|
| 120 |
+
out[idx] = x * 0.5f * (1.0f + erff(x * 0.7071067811865475f));
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
// -------------------------------------------------------------
|
| 124 |
+
// 4. SiLU Activation Kernel
|
| 125 |
+
// -------------------------------------------------------------
|
| 126 |
+
__global__ void silu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) {
|
| 127 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 128 |
+
if (idx >= N) return;
|
| 129 |
+
float x = in[idx];
|
| 130 |
+
out[idx] = x / (1.0f + expf(-x));
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
// -------------------------------------------------------------
|
| 134 |
+
// 5. FiLM Modulation Kernel: out = norm_h * (1.0 + scale) + shift
|
| 135 |
+
// -------------------------------------------------------------
|
| 136 |
+
__global__ void film_modulate_256_kernel(
|
| 137 |
+
const float* __restrict__ h_norm,
|
| 138 |
+
const float* __restrict__ film_cond, // [512] -> scale (0:256), shift (256:512)
|
| 139 |
+
float* __restrict__ out,
|
| 140 |
+
int num_tokens
|
| 141 |
+
) {
|
| 142 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 143 |
+
if (idx >= num_tokens * 256) return;
|
| 144 |
+
|
| 145 |
+
int d = idx % 256;
|
| 146 |
+
|
| 147 |
+
float scale = film_cond[d];
|
| 148 |
+
float shift = film_cond[256 + d];
|
| 149 |
+
|
| 150 |
+
out[idx] = h_norm[idx] * (1.0f + scale) + shift;
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
// -------------------------------------------------------------
|
| 154 |
+
// 6. Softmax Kernel (num_rows, num_cols <= 32)
|
| 155 |
+
// -------------------------------------------------------------
|
| 156 |
+
__global__ void softmax_small_kernel(
|
| 157 |
+
float* __restrict__ data,
|
| 158 |
+
int num_rows,
|
| 159 |
+
int num_cols
|
| 160 |
+
) {
|
| 161 |
+
int row = blockIdx.x;
|
| 162 |
+
if (row >= num_rows) return;
|
| 163 |
+
|
| 164 |
+
int tid = threadIdx.x;
|
| 165 |
+
float* row_data = data + row * num_cols;
|
| 166 |
+
|
| 167 |
+
float val = (tid < num_cols) ? row_data[tid] : -1e20f;
|
| 168 |
+
|
| 169 |
+
// Max reduction
|
| 170 |
+
float max_val = val;
|
| 171 |
+
#pragma unroll
|
| 172 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 173 |
+
max_val = fmaxf(max_val, __shfl_down_sync(0xffffffff, max_val, offset));
|
| 174 |
+
}
|
| 175 |
+
max_val = __shfl_sync(0xffffffff, max_val, 0);
|
| 176 |
+
|
| 177 |
+
float exp_val = (tid < num_cols) ? expf(val - max_val) : 0.0f;
|
| 178 |
+
|
| 179 |
+
// Sum reduction
|
| 180 |
+
float sum = exp_val;
|
| 181 |
+
#pragma unroll
|
| 182 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 183 |
+
sum += __shfl_down_sync(0xffffffff, sum, offset);
|
| 184 |
+
}
|
| 185 |
+
sum = __shfl_sync(0xffffffff, sum, 0);
|
| 186 |
+
|
| 187 |
+
if (tid < num_cols) {
|
| 188 |
+
row_data[tid] = exp_val / (sum + 1e-12f);
|
| 189 |
+
}
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
// -------------------------------------------------------------
|
| 193 |
+
// 7. Euler Integration Step: z = z + dt * v
|
| 194 |
+
// -------------------------------------------------------------
|
| 195 |
+
__global__ void euler_step_kernel(
|
| 196 |
+
float* __restrict__ z,
|
| 197 |
+
const float* __restrict__ v,
|
| 198 |
+
float dt,
|
| 199 |
+
int N
|
| 200 |
+
) {
|
| 201 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 202 |
+
if (idx >= N) return;
|
| 203 |
+
z[idx] += dt * v[idx];
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
// -------------------------------------------------------------
|
| 207 |
+
// 8. Action Denormalization: a = ((z + 1)/2) * (max - min) + min
|
| 208 |
+
// -------------------------------------------------------------
|
| 209 |
+
__global__ void denormalize_actions_kernel(
|
| 210 |
+
const float* __restrict__ z_chunk,
|
| 211 |
+
const float* __restrict__ act_min,
|
| 212 |
+
const float* __restrict__ act_max,
|
| 213 |
+
float* __restrict__ out_actions,
|
| 214 |
+
int chunk_size,
|
| 215 |
+
int act_dim
|
| 216 |
+
) {
|
| 217 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 218 |
+
if (idx >= chunk_size * act_dim) return;
|
| 219 |
+
|
| 220 |
+
int dim_idx = idx % act_dim;
|
| 221 |
+
float z = z_chunk[idx];
|
| 222 |
+
float mn = act_min[dim_idx];
|
| 223 |
+
float mx = act_max[dim_idx];
|
| 224 |
+
|
| 225 |
+
// Standard denormalization
|
| 226 |
+
out_actions[idx] = ((z + 1.0f) * 0.5f) * (mx - mn) + mn;
|
| 227 |
+
}
|
| 228 |
+
|
| 229 |
+
// -------------------------------------------------------------
|
| 230 |
+
// 9. Sinusoidal Timestep Embedding (dim=256)
|
| 231 |
+
// -------------------------------------------------------------
|
| 232 |
+
__global__ void sinusoidal_embedding_kernel(float t, float* __restrict__ out_emb, int dim = 256) {
|
| 233 |
+
int tid = threadIdx.x;
|
| 234 |
+
if (tid >= dim / 2) return;
|
| 235 |
+
|
| 236 |
+
int half = dim / 2;
|
| 237 |
+
float freq = expf(-logf(10000.0f) * (float)tid / (float)half);
|
| 238 |
+
float arg = t * freq * 1000.0f;
|
| 239 |
+
|
| 240 |
+
out_emb[tid] = sinf(arg);
|
| 241 |
+
out_emb[half + tid] = cosf(arg);
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
// -------------------------------------------------------------
|
| 245 |
+
// 10. Transpose Packed QKV to Head-Major
|
| 246 |
+
// -------------------------------------------------------------
|
| 247 |
+
__global__ void transpose_qkv_to_heads_kernel(
|
| 248 |
+
const float* __restrict__ qkv,
|
| 249 |
+
float* __restrict__ q_head,
|
| 250 |
+
float* __restrict__ k_head,
|
| 251 |
+
float* __restrict__ v_head,
|
| 252 |
+
int N, int H, int D
|
| 253 |
+
) {
|
| 254 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 255 |
+
int total = N * H * D;
|
| 256 |
+
if (idx >= total) return;
|
| 257 |
+
int d = idx % D;
|
| 258 |
+
int h = (idx / D) % H;
|
| 259 |
+
int n = idx / (H * D);
|
| 260 |
+
|
| 261 |
+
int row_stride = 3 * H * D;
|
| 262 |
+
int hd = H * D;
|
| 263 |
+
|
| 264 |
+
int q_in = n * row_stride + h * D + d;
|
| 265 |
+
int k_in = n * row_stride + hd + h * D + d;
|
| 266 |
+
int v_in = n * row_stride + 2 * hd + h * D + d;
|
| 267 |
+
|
| 268 |
+
int out_idx = h * (N * D) + n * D + d;
|
| 269 |
+
q_head[out_idx] = qkv[q_in];
|
| 270 |
+
k_head[out_idx] = qkv[k_in];
|
| 271 |
+
v_head[out_idx] = qkv[v_in];
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
// -------------------------------------------------------------
|
| 275 |
+
// 11. Clamp Kernel
|
| 276 |
+
// -------------------------------------------------------------
|
| 277 |
+
__global__ void clamp_kernel(float* data, float min_val, float max_val, int N) {
|
| 278 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 279 |
+
if (idx < N) {
|
| 280 |
+
data[idx] = fminf(max_val, fmaxf(min_val, data[idx]));
|
| 281 |
+
}
|
| 282 |
+
}
|
c_engine/parvulus_c_api.cpp
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "parvulus_c_api.h"
|
| 2 |
+
#include "parvulus_engine.h"
|
| 3 |
+
#include <cstring>
|
| 4 |
+
|
| 5 |
+
extern "C" {
|
| 6 |
+
|
| 7 |
+
ParvulusHandle parvulus_create(int device_id) {
|
| 8 |
+
return new ParvulusEngine(device_id);
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
bool parvulus_load(
|
| 12 |
+
ParvulusHandle handle,
|
| 13 |
+
const char* weights_path,
|
| 14 |
+
const char* embeddings_path,
|
| 15 |
+
const char* scale_path
|
| 16 |
+
) {
|
| 17 |
+
if (!handle) return false;
|
| 18 |
+
ParvulusEngine* engine = static_cast<ParvulusEngine*>(handle);
|
| 19 |
+
bool ok = engine->load_weights(weights_path);
|
| 20 |
+
if (embeddings_path != nullptr && std::strlen(embeddings_path) > 0) {
|
| 21 |
+
ok = ok && engine->load_task_embeddings(embeddings_path);
|
| 22 |
+
}
|
| 23 |
+
if (scale_path != nullptr && std::strlen(scale_path) > 0) {
|
| 24 |
+
ok = ok && engine->load_action_scale(scale_path);
|
| 25 |
+
}
|
| 26 |
+
return ok;
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
bool parvulus_set_task_vector(ParvulusHandle handle, const float* task_vector) {
|
| 30 |
+
if (!handle || !task_vector) return false;
|
| 31 |
+
ParvulusEngine* engine = static_cast<ParvulusEngine*>(handle);
|
| 32 |
+
return engine->set_task_vector(task_vector);
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
bool parvulus_set_task_string(ParvulusHandle handle, const char* task_description) {
|
| 36 |
+
if (!handle || !task_description) return false;
|
| 37 |
+
ParvulusEngine* engine = static_cast<ParvulusEngine*>(handle);
|
| 38 |
+
return engine->set_task_string(task_description);
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
bool parvulus_predict_chunk(
|
| 42 |
+
ParvulusHandle handle,
|
| 43 |
+
const uint8_t* agent_img,
|
| 44 |
+
const uint8_t* wrist_img,
|
| 45 |
+
const char* task_description,
|
| 46 |
+
float* out_actions,
|
| 47 |
+
int num_euler_steps,
|
| 48 |
+
const float* custom_z0
|
| 49 |
+
) {
|
| 50 |
+
if (!handle) return false;
|
| 51 |
+
ParvulusEngine* engine = static_cast<ParvulusEngine*>(handle);
|
| 52 |
+
if (task_description != nullptr && std::strlen(task_description) > 0) {
|
| 53 |
+
return engine->predict_action_chunk(agent_img, wrist_img, task_description, out_actions, num_euler_steps, custom_z0);
|
| 54 |
+
} else {
|
| 55 |
+
return engine->predict_action_chunk(agent_img, wrist_img, out_actions, num_euler_steps, custom_z0);
|
| 56 |
+
}
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
bool parvulus_predict_chunk_with_vector(
|
| 60 |
+
ParvulusHandle handle,
|
| 61 |
+
const uint8_t* agent_img,
|
| 62 |
+
const uint8_t* wrist_img,
|
| 63 |
+
const float* task_vector,
|
| 64 |
+
float* out_actions,
|
| 65 |
+
int num_euler_steps,
|
| 66 |
+
const float* custom_z0
|
| 67 |
+
) {
|
| 68 |
+
if (!handle) return false;
|
| 69 |
+
ParvulusEngine* engine = static_cast<ParvulusEngine*>(handle);
|
| 70 |
+
return engine->predict_action_chunk_with_vector(agent_img, wrist_img, task_vector, out_actions, num_euler_steps, custom_z0);
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
bool parvulus_predict_chunk_with_emb(
|
| 74 |
+
ParvulusHandle handle,
|
| 75 |
+
const uint8_t* agent_img,
|
| 76 |
+
const uint8_t* wrist_img,
|
| 77 |
+
const float* text_embedding,
|
| 78 |
+
float* out_actions,
|
| 79 |
+
int num_euler_steps,
|
| 80 |
+
const float* custom_z0
|
| 81 |
+
) {
|
| 82 |
+
return parvulus_predict_chunk_with_vector(handle, agent_img, wrist_img, text_embedding, out_actions, num_euler_steps, custom_z0);
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
bool parvulus_register_task_embedding(
|
| 86 |
+
ParvulusHandle handle,
|
| 87 |
+
const char* task_description,
|
| 88 |
+
const float* text_embedding
|
| 89 |
+
) {
|
| 90 |
+
if (!handle || !task_description || !text_embedding) return false;
|
| 91 |
+
ParvulusEngine* engine = static_cast<ParvulusEngine*>(handle);
|
| 92 |
+
return engine->register_task_embedding(task_description, text_embedding);
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
void parvulus_destroy(ParvulusHandle handle) {
|
| 96 |
+
if (handle) {
|
| 97 |
+
delete static_cast<ParvulusEngine*>(handle);
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
}
|
c_engine/parvulus_c_api.h
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
/**
|
| 3 |
+
* @file parvulus_c_api.h
|
| 4 |
+
* @brief Pure C ABI interface for high-frequency PARVULUS robot policy inference.
|
| 5 |
+
*
|
| 6 |
+
* Standalone native CUDA/cuBLAS runtime with ZERO PyTorch, LibTorch, or Python dependencies.
|
| 7 |
+
* Designed for high-frequency (136 Hz) physical robot manipulation loops (ROS 2, FCI, MuJoCo).
|
| 8 |
+
*/
|
| 9 |
+
|
| 10 |
+
#include <stdint.h>
|
| 11 |
+
#include <stdbool.h>
|
| 12 |
+
|
| 13 |
+
#ifdef __cplusplus
|
| 14 |
+
extern "C" {
|
| 15 |
+
#endif
|
| 16 |
+
|
| 17 |
+
typedef void* ParvulusHandle;
|
| 18 |
+
|
| 19 |
+
/**
|
| 20 |
+
* @brief Initializes a PARVULUS engine context on the designated CUDA device.
|
| 21 |
+
* @param device_id CUDA device index (0, 1, 2, ...).
|
| 22 |
+
* @return Opaque handle to the initialized engine, or NULL on error.
|
| 23 |
+
*/
|
| 24 |
+
ParvulusHandle parvulus_create(int device_id);
|
| 25 |
+
|
| 26 |
+
/**
|
| 27 |
+
* @brief Loads model weights, optional task language representations, and normalization scales.
|
| 28 |
+
* @param handle Valid ParvulusHandle.
|
| 29 |
+
* @param weights_path Path to flat model weights binary (parvulus_model_weights.bin).
|
| 30 |
+
* @param embeddings_path Path to pre-cached task language embeddings binary (optional, can be NULL).
|
| 31 |
+
* @param scale_path Path to 7-DoF action scale binary (optional, can be NULL).
|
| 32 |
+
* @return true if loading succeeded, false otherwise.
|
| 33 |
+
*/
|
| 34 |
+
bool parvulus_load(
|
| 35 |
+
ParvulusHandle handle,
|
| 36 |
+
const char* weights_path,
|
| 37 |
+
const char* embeddings_path,
|
| 38 |
+
const char* scale_path
|
| 39 |
+
);
|
| 40 |
+
|
| 41 |
+
/**
|
| 42 |
+
* @brief Sets the active 256-dim task conditioning vector on GPU VRAM.
|
| 43 |
+
* All subsequent chunk predictions will use this vector until updated.
|
| 44 |
+
*
|
| 45 |
+
* @param handle Valid ParvulusHandle.
|
| 46 |
+
* @param task_vector Array of 256 float32 values.
|
| 47 |
+
* @return true if successfully uploaded to GPU, false otherwise.
|
| 48 |
+
*/
|
| 49 |
+
bool parvulus_set_task_vector(ParvulusHandle handle, const float* task_vector);
|
| 50 |
+
|
| 51 |
+
/**
|
| 52 |
+
* @brief Sets the active task conditioning from a pre-cached task description string.
|
| 53 |
+
*
|
| 54 |
+
* @param handle Valid ParvulusHandle.
|
| 55 |
+
* @param task_description String matching an entry in task_embeddings.bin.
|
| 56 |
+
* @return true if found and set, false if string was not in cache.
|
| 57 |
+
*/
|
| 58 |
+
bool parvulus_set_task_string(ParvulusHandle handle, const char* task_description);
|
| 59 |
+
|
| 60 |
+
/**
|
| 61 |
+
* @brief Generates an 8-step continuous action chunk from dual 256x256 RGB observations.
|
| 62 |
+
*
|
| 63 |
+
* @param handle Valid ParvulusHandle.
|
| 64 |
+
* @param agent_img Contiguous [256, 256, 3] uint8 RGB camera frame (agent POV).
|
| 65 |
+
* @param wrist_img Contiguous [256, 256, 3] uint8 RGB camera frame (eye-in-hand POV).
|
| 66 |
+
* @param task_description Optional task string (pass NULL to use active task conditioning vector).
|
| 67 |
+
* @param out_actions Output buffer of length 56 (8 steps x 7 DoF: dx, dy, dz, drx, dry, drz, gripper).
|
| 68 |
+
* @param num_euler_steps Number of ODE flow integration steps (typically 4 or 10).
|
| 69 |
+
* @param custom_z0 Optional initial noise buffer [56 floats] for deterministic testing (NULL for random).
|
| 70 |
+
* @return true if prediction succeeded, false on error.
|
| 71 |
+
*/
|
| 72 |
+
bool parvulus_predict_chunk(
|
| 73 |
+
ParvulusHandle handle,
|
| 74 |
+
const uint8_t* agent_img,
|
| 75 |
+
const uint8_t* wrist_img,
|
| 76 |
+
const char* task_description,
|
| 77 |
+
float* out_actions,
|
| 78 |
+
int num_euler_steps,
|
| 79 |
+
const float* custom_z0
|
| 80 |
+
);
|
| 81 |
+
|
| 82 |
+
/**
|
| 83 |
+
* @brief Predicts an action chunk directly using a 256-dim task conditioning vector.
|
| 84 |
+
* Updates the active task vector on GPU and generates the action chunk.
|
| 85 |
+
*/
|
| 86 |
+
bool parvulus_predict_chunk_with_vector(
|
| 87 |
+
ParvulusHandle handle,
|
| 88 |
+
const uint8_t* agent_img,
|
| 89 |
+
const uint8_t* wrist_img,
|
| 90 |
+
const float* task_vector,
|
| 91 |
+
float* out_actions,
|
| 92 |
+
int num_euler_steps,
|
| 93 |
+
const float* custom_z0
|
| 94 |
+
);
|
| 95 |
+
|
| 96 |
+
/**
|
| 97 |
+
* @brief Alias for parvulus_predict_chunk_with_vector.
|
| 98 |
+
*/
|
| 99 |
+
bool parvulus_predict_chunk_with_emb(
|
| 100 |
+
ParvulusHandle handle,
|
| 101 |
+
const uint8_t* agent_img,
|
| 102 |
+
const uint8_t* wrist_img,
|
| 103 |
+
const float* text_embedding,
|
| 104 |
+
float* out_actions,
|
| 105 |
+
int num_euler_steps,
|
| 106 |
+
const float* custom_z0
|
| 107 |
+
);
|
| 108 |
+
|
| 109 |
+
/**
|
| 110 |
+
* @brief Registers or updates a task description string and its 256-dim embedding dynamically in memory.
|
| 111 |
+
*/
|
| 112 |
+
bool parvulus_register_task_embedding(
|
| 113 |
+
ParvulusHandle handle,
|
| 114 |
+
const char* task_description,
|
| 115 |
+
const float* text_embedding
|
| 116 |
+
);
|
| 117 |
+
|
| 118 |
+
/**
|
| 119 |
+
* @brief Destroys engine instance and releases all GPU allocations.
|
| 120 |
+
*/
|
| 121 |
+
void parvulus_destroy(ParvulusHandle handle);
|
| 122 |
+
|
| 123 |
+
#ifdef __cplusplus
|
| 124 |
+
}
|
| 125 |
+
#endif
|
c_engine/parvulus_engine.cu
ADDED
|
@@ -0,0 +1,686 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "parvulus_engine.h"
|
| 2 |
+
#include "kernels.cuh"
|
| 3 |
+
#include "vision_encoder.cu"
|
| 4 |
+
|
| 5 |
+
#include <iostream>
|
| 6 |
+
#include <fstream>
|
| 7 |
+
#include <cstring>
|
| 8 |
+
#include <cmath>
|
| 9 |
+
#include <random>
|
| 10 |
+
|
| 11 |
+
ParvulusEngine::ParvulusEngine(int device_id) : device_id_(device_id) {
|
| 12 |
+
cudaSetDevice(device_id_);
|
| 13 |
+
cudaStreamCreate(&stream_);
|
| 14 |
+
cublasCreate(&cublas_handle_);
|
| 15 |
+
cublasSetStream(cublas_handle_, stream_);
|
| 16 |
+
rng_.seed(42 + device_id_);
|
| 17 |
+
allocate_scratchpads();
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
ParvulusEngine::~ParvulusEngine() {
|
| 21 |
+
free_scratchpads();
|
| 22 |
+
if (ode_graph_exec_) {
|
| 23 |
+
cudaGraphExecDestroy(ode_graph_exec_);
|
| 24 |
+
cudaGraphDestroy(ode_graph_);
|
| 25 |
+
}
|
| 26 |
+
if (d_weights_buffer_) cudaFree(d_weights_buffer_);
|
| 27 |
+
if (d_text_emb_) cudaFree(d_text_emb_);
|
| 28 |
+
if (d_action_min_) cudaFree(d_action_min_);
|
| 29 |
+
if (d_action_max_) cudaFree(d_action_max_);
|
| 30 |
+
cublasDestroy(cublas_handle_);
|
| 31 |
+
cudaStreamDestroy(stream_);
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
void ParvulusEngine::allocate_scratchpads() {
|
| 35 |
+
// Scratchpads for zero-allocation runtime
|
| 36 |
+
cudaMalloc(&d_obs_, 6 * 256 * 256 * sizeof(float));
|
| 37 |
+
cudaMalloc(&d_z_vis_, 16 * 256 * sizeof(float));
|
| 38 |
+
cudaMalloc(&d_z_sigma_, 8 * 7 * sizeof(float));
|
| 39 |
+
cudaMalloc(&d_v_pred_, 8 * 7 * sizeof(float));
|
| 40 |
+
cudaMalloc(&d_cond_, 256 * sizeof(float));
|
| 41 |
+
cudaMalloc(&d_t_emb_, 256 * sizeof(float));
|
| 42 |
+
|
| 43 |
+
// Large vision activation buffers (384x128x128 max)
|
| 44 |
+
cudaMalloc(&d_vis_scratch_a_, 384 * 128 * 128 * sizeof(float));
|
| 45 |
+
cudaMalloc(&d_vis_scratch_b_, 384 * 128 * 128 * sizeof(float));
|
| 46 |
+
cudaMalloc(&d_vis_scratch_c_, 384 * 128 * 128 * sizeof(float));
|
| 47 |
+
cudaMalloc(&d_vis_scratch_d_, 384 * 128 * 128 * sizeof(float));
|
| 48 |
+
cudaMalloc(&d_im2col_, 192 * 9 * 128 * 128 * sizeof(float));
|
| 49 |
+
cudaMalloc(&d_film_ss_, 1024 * sizeof(float));
|
| 50 |
+
cudaMalloc(&d_pool_k_, 16384 * 256 * sizeof(float));
|
| 51 |
+
cudaMalloc(&d_pool_v_, 16384 * 256 * sizeof(float));
|
| 52 |
+
cudaMalloc(&d_pool_k_head_, 16384 * 256 * sizeof(float));
|
| 53 |
+
cudaMalloc(&d_pool_v_head_, 16384 * 256 * sizeof(float));
|
| 54 |
+
cudaMalloc(&d_pool_q_, 4 * 256 * sizeof(float));
|
| 55 |
+
cudaMalloc(&d_pool_q_head_, 4 * 256 * sizeof(float));
|
| 56 |
+
cudaMalloc(&d_pool_scores_, 4 * 4 * 16384 * sizeof(float));
|
| 57 |
+
cudaMalloc(&d_pool_ctx_head_, 4 * 256 * sizeof(float));
|
| 58 |
+
cudaMalloc(&d_pool_context_, 4 * 256 * sizeof(float));
|
| 59 |
+
cudaMalloc(&d_gn_stats_, 16 * sizeof(float));
|
| 60 |
+
|
| 61 |
+
int stage_in_ch[4] = {96, 192, 192, 384};
|
| 62 |
+
for (int s = 0; s < 4; ++s) {
|
| 63 |
+
int in_c = stage_in_ch[s];
|
| 64 |
+
cudaMalloc(&d_attn_pool_wk_fused_[s], 256 * in_c * sizeof(float));
|
| 65 |
+
cudaMalloc(&d_attn_pool_bk_fused_[s], 256 * sizeof(float));
|
| 66 |
+
cudaMalloc(&d_attn_pool_wv_fused_[s], 256 * in_c * sizeof(float));
|
| 67 |
+
cudaMalloc(&d_attn_pool_bv_fused_[s], 256 * sizeof(float));
|
| 68 |
+
cudaMalloc(&d_attn_pool_q_head_[s], 4 * 4 * 64 * sizeof(float));
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
// Transformer scratchpads
|
| 72 |
+
cudaMalloc(&d_trans_h_, 8 * 256 * sizeof(float));
|
| 73 |
+
cudaMalloc(&d_trans_norm_, 8 * 256 * sizeof(float));
|
| 74 |
+
cudaMalloc(&d_trans_qkv_, 8 * 768 * sizeof(float));
|
| 75 |
+
cudaMalloc(&d_trans_q_head_, 8 * 8 * 32 * sizeof(float));
|
| 76 |
+
cudaMalloc(&d_trans_k_head_, 8 * 16 * 32 * sizeof(float));
|
| 77 |
+
cudaMalloc(&d_trans_v_head_, 8 * 16 * 32 * sizeof(float));
|
| 78 |
+
cudaMalloc(&d_trans_scores_, 8 * 8 * 16 * sizeof(float));
|
| 79 |
+
cudaMalloc(&d_trans_ctx_head_, 8 * 8 * 32 * sizeof(float));
|
| 80 |
+
cudaMalloc(&d_trans_context_, 8 * 256 * sizeof(float));
|
| 81 |
+
cudaMalloc(&d_trans_out_, 8 * 256 * sizeof(float));
|
| 82 |
+
cudaMalloc(&d_trans_mlp_, 8 * 1024 * sizeof(float));
|
| 83 |
+
cudaMalloc(&d_trans_film_, 512 * sizeof(float));
|
| 84 |
+
cudaMalloc(&d_temp_k_, 16 * 256 * sizeof(float));
|
| 85 |
+
cudaMalloc(&d_temp_v_, 16 * 256 * sizeof(float));
|
| 86 |
+
|
| 87 |
+
// Precomputed Visual KV Cache scratchpads (4 layers x 8 heads x 16 keys x 32 dims)
|
| 88 |
+
cudaMalloc(&d_cached_k_, 4 * 8 * 16 * 32 * sizeof(float));
|
| 89 |
+
cudaMalloc(&d_cached_v_, 4 * 8 * 16 * 32 * sizeof(float));
|
| 90 |
+
cudaMalloc(&d_q_proj_, 8 * 256 * sizeof(float));
|
| 91 |
+
|
| 92 |
+
cudaMalloc(&d_action_min_, 7 * sizeof(float));
|
| 93 |
+
cudaMalloc(&d_action_max_, 7 * sizeof(float));
|
| 94 |
+
cudaMalloc(&d_text_emb_, 256 * sizeof(float));
|
| 95 |
+
|
| 96 |
+
cudaMalloc(&d_raw_agent_, 256 * 256 * 3);
|
| 97 |
+
cudaMalloc(&d_raw_wrist_, 256 * 256 * 3);
|
| 98 |
+
cudaMalloc(&d_final_actions_, 8 * 7 * sizeof(float));
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
void ParvulusEngine::free_scratchpads() {
|
| 102 |
+
cudaFree(d_obs_);
|
| 103 |
+
cudaFree(d_z_vis_);
|
| 104 |
+
cudaFree(d_z_sigma_);
|
| 105 |
+
cudaFree(d_v_pred_);
|
| 106 |
+
cudaFree(d_cond_);
|
| 107 |
+
cudaFree(d_t_emb_);
|
| 108 |
+
cudaFree(d_vis_scratch_a_);
|
| 109 |
+
cudaFree(d_vis_scratch_b_);
|
| 110 |
+
cudaFree(d_vis_scratch_c_);
|
| 111 |
+
cudaFree(d_vis_scratch_d_);
|
| 112 |
+
cudaFree(d_im2col_);
|
| 113 |
+
cudaFree(d_film_ss_);
|
| 114 |
+
cudaFree(d_pool_k_);
|
| 115 |
+
cudaFree(d_pool_v_);
|
| 116 |
+
cudaFree(d_pool_k_head_);
|
| 117 |
+
cudaFree(d_pool_v_head_);
|
| 118 |
+
cudaFree(d_pool_q_);
|
| 119 |
+
cudaFree(d_pool_q_head_);
|
| 120 |
+
cudaFree(d_pool_scores_);
|
| 121 |
+
cudaFree(d_pool_ctx_head_);
|
| 122 |
+
cudaFree(d_pool_context_);
|
| 123 |
+
if (d_gn_stats_) cudaFree(d_gn_stats_);
|
| 124 |
+
for (int s = 0; s < 4; ++s) {
|
| 125 |
+
if (d_attn_pool_wk_fused_[s]) cudaFree(d_attn_pool_wk_fused_[s]);
|
| 126 |
+
if (d_attn_pool_bk_fused_[s]) cudaFree(d_attn_pool_bk_fused_[s]);
|
| 127 |
+
if (d_attn_pool_wv_fused_[s]) cudaFree(d_attn_pool_wv_fused_[s]);
|
| 128 |
+
if (d_attn_pool_bv_fused_[s]) cudaFree(d_attn_pool_bv_fused_[s]);
|
| 129 |
+
if (d_attn_pool_q_head_[s]) cudaFree(d_attn_pool_q_head_[s]);
|
| 130 |
+
}
|
| 131 |
+
cudaFree(d_trans_h_);
|
| 132 |
+
cudaFree(d_trans_norm_);
|
| 133 |
+
cudaFree(d_trans_qkv_);
|
| 134 |
+
cudaFree(d_trans_q_head_);
|
| 135 |
+
cudaFree(d_trans_k_head_);
|
| 136 |
+
cudaFree(d_trans_v_head_);
|
| 137 |
+
cudaFree(d_trans_scores_);
|
| 138 |
+
cudaFree(d_trans_ctx_head_);
|
| 139 |
+
cudaFree(d_trans_context_);
|
| 140 |
+
cudaFree(d_trans_out_);
|
| 141 |
+
cudaFree(d_trans_mlp_);
|
| 142 |
+
cudaFree(d_trans_film_);
|
| 143 |
+
cudaFree(d_temp_k_);
|
| 144 |
+
cudaFree(d_temp_v_);
|
| 145 |
+
cudaFree(d_cached_k_);
|
| 146 |
+
cudaFree(d_cached_v_);
|
| 147 |
+
cudaFree(d_q_proj_);
|
| 148 |
+
cudaFree(d_raw_agent_);
|
| 149 |
+
cudaFree(d_raw_wrist_);
|
| 150 |
+
cudaFree(d_final_actions_);
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
bool ParvulusEngine::load_weights(const std::string& weights_path) {
|
| 154 |
+
std::ifstream file(weights_path, std::ios::binary);
|
| 155 |
+
if (!file.is_open()) {
|
| 156 |
+
std::cerr << "[ParvulusEngine] Failed to open weights file: " << weights_path << std::endl;
|
| 157 |
+
return false;
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
uint32_t magic, version, num_tensors, header_size;
|
| 161 |
+
file.read(reinterpret_cast<char*>(&magic), 4);
|
| 162 |
+
file.read(reinterpret_cast<char*>(&version), 4);
|
| 163 |
+
file.read(reinterpret_cast<char*>(&num_tensors), 4);
|
| 164 |
+
file.read(reinterpret_cast<char*>(&header_size), 4);
|
| 165 |
+
|
| 166 |
+
if (magic != 0x54524B33 && magic != 0x50415256) {
|
| 167 |
+
std::cerr << "[ParvulusEngine] Invalid magic number in weight binary!" << std::endl;
|
| 168 |
+
return false;
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
tensors_.clear();
|
| 172 |
+
size_t data_section_size = 0;
|
| 173 |
+
|
| 174 |
+
for (uint32_t i = 0; i < num_tensors; ++i) {
|
| 175 |
+
uint16_t name_len;
|
| 176 |
+
file.read(reinterpret_cast<char*>(&name_len), 2);
|
| 177 |
+
std::string name(name_len, '\0');
|
| 178 |
+
file.read(&name[0], name_len);
|
| 179 |
+
|
| 180 |
+
uint16_t num_dims;
|
| 181 |
+
uint32_t dims[4];
|
| 182 |
+
uint64_t offset, size_bytes;
|
| 183 |
+
file.read(reinterpret_cast<char*>(&num_dims), 2);
|
| 184 |
+
file.read(reinterpret_cast<char*>(dims), 4 * 4);
|
| 185 |
+
file.read(reinterpret_cast<char*>(&offset), 8);
|
| 186 |
+
file.read(reinterpret_cast<char*>(&size_bytes), 8);
|
| 187 |
+
|
| 188 |
+
TensorInfo info;
|
| 189 |
+
info.name = name;
|
| 190 |
+
for (int d = 0; d < num_dims; ++d) info.shape.push_back(dims[d]);
|
| 191 |
+
info.offset = offset;
|
| 192 |
+
info.size_bytes = size_bytes;
|
| 193 |
+
tensors_[name] = info;
|
| 194 |
+
|
| 195 |
+
data_section_size = std::max(data_section_size, (size_t)(offset + size_bytes));
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
total_weight_bytes_ = data_section_size;
|
| 199 |
+
std::vector<char> h_weights_data(data_section_size);
|
| 200 |
+
file.read(h_weights_data.data(), data_section_size);
|
| 201 |
+
|
| 202 |
+
cudaMalloc(&d_weights_buffer_, total_weight_bytes_);
|
| 203 |
+
cudaMemcpy(d_weights_buffer_, h_weights_data.data(), total_weight_bytes_, cudaMemcpyHostToDevice);
|
| 204 |
+
|
| 205 |
+
// Map device pointers
|
| 206 |
+
for (auto& pair : tensors_) {
|
| 207 |
+
pair.second.d_ptr = reinterpret_cast<float*>(reinterpret_cast<char*>(d_weights_buffer_) + pair.second.offset);
|
| 208 |
+
}
|
| 209 |
+
|
| 210 |
+
// Precompute / Fuse AttnPool weights and static query heads for all 4 stages
|
| 211 |
+
int stage_in_ch[4] = {96, 192, 192, 384};
|
| 212 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 213 |
+
for (int s = 0; s < 4; ++s) {
|
| 214 |
+
int in_c = stage_in_ch[s];
|
| 215 |
+
std::string pfx = "encoder.attn_poolers." + std::to_string(s) + ".";
|
| 216 |
+
float* w_ip = get_tensor(pfx + "in_proj.weight");
|
| 217 |
+
float* b_ip = get_tensor(pfx + "in_proj.bias");
|
| 218 |
+
float* w_qkv = get_tensor(pfx + "cross_attn.in_proj_weight");
|
| 219 |
+
float* b_qkv = get_tensor(pfx + "cross_attn.in_proj_bias");
|
| 220 |
+
float* q_in = get_tensor(pfx + "queries");
|
| 221 |
+
|
| 222 |
+
float* w_q = w_qkv;
|
| 223 |
+
float* b_q = b_qkv;
|
| 224 |
+
float* w_k = w_qkv + 256 * 256;
|
| 225 |
+
float* b_k = b_qkv + 256;
|
| 226 |
+
float* w_v = w_qkv + 512 * 256;
|
| 227 |
+
float* b_v = b_qkv + 512;
|
| 228 |
+
|
| 229 |
+
// 1. Fused W_K: in_c x 256 col-major
|
| 230 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_k, 256, &beta, d_attn_pool_wk_fused_[s], in_c);
|
| 231 |
+
// Fused b_K: W_K * b_ip + b_K
|
| 232 |
+
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_k, 256, b_ip, 1, &beta, d_attn_pool_bk_fused_[s], 1);
|
| 233 |
+
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bk_fused_[s], b_k, 256);
|
| 234 |
+
|
| 235 |
+
// 2. Fused W_V: in_c x 256 col-major
|
| 236 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_N, CUBLAS_OP_N, in_c, 256, 256, &alpha, w_ip, in_c, w_v, 256, &beta, d_attn_pool_wv_fused_[s], in_c);
|
| 237 |
+
// Fused b_V: W_V * b_ip + b_V
|
| 238 |
+
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w_v, 256, b_ip, 1, &beta, d_attn_pool_bv_fused_[s], 1);
|
| 239 |
+
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_attn_pool_bv_fused_[s], b_v, 256);
|
| 240 |
+
|
| 241 |
+
// 3. Precompute Q_head:
|
| 242 |
+
// q_in [4, 256] * W_Q^T + b_Q -> temp in d_pool_q_
|
| 243 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_q, 256, q_in, 256, &beta, d_pool_q_, 256);
|
| 244 |
+
add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream_>>>(d_pool_q_, b_q, 4, 256);
|
| 245 |
+
transpose_interleaved_to_head_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream_>>>(
|
| 246 |
+
d_pool_q_, d_attn_pool_q_head_[s], 4, 4, 64
|
| 247 |
+
);
|
| 248 |
+
}
|
| 249 |
+
cudaStreamSynchronize(stream_);
|
| 250 |
+
|
| 251 |
+
std::cout << "[ParvulusEngine] Successfully loaded " << num_tensors << " weight tensors ("
|
| 252 |
+
<< (total_weight_bytes_ / (1024.0 * 1024.0)) << " MB) into GPU VRAM." << std::endl;
|
| 253 |
+
return true;
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
bool ParvulusEngine::load_task_embeddings(const std::string& embeddings_path) {
|
| 257 |
+
std::ifstream file(embeddings_path, std::ios::binary);
|
| 258 |
+
if (!file.is_open()) {
|
| 259 |
+
std::cerr << "[ParvulusEngine] Failed to open task embeddings: " << embeddings_path << std::endl;
|
| 260 |
+
return false;
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
uint32_t num_tasks;
|
| 264 |
+
file.read(reinterpret_cast<char*>(&num_tasks), 4);
|
| 265 |
+
task_embeddings_.clear();
|
| 266 |
+
|
| 267 |
+
for (uint32_t i = 0; i < num_tasks; ++i) {
|
| 268 |
+
uint16_t str_len;
|
| 269 |
+
file.read(reinterpret_cast<char*>(&str_len), 2);
|
| 270 |
+
std::string task_str(str_len, '\0');
|
| 271 |
+
file.read(&task_str[0], str_len);
|
| 272 |
+
|
| 273 |
+
std::vector<float> emb(256);
|
| 274 |
+
file.read(reinterpret_cast<char*>(emb.data()), 256 * sizeof(float));
|
| 275 |
+
task_embeddings_[task_str] = emb;
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
std::cout << "[ParvulusEngine] Loaded " << num_tasks << " pre-cached task language embeddings." << std::endl;
|
| 279 |
+
return true;
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
bool ParvulusEngine::load_action_scale(const std::string& scale_path) {
|
| 283 |
+
std::ifstream file(scale_path, std::ios::binary);
|
| 284 |
+
if (!file.is_open()) {
|
| 285 |
+
std::cerr << "[ParvulusEngine] Failed to open action scale: " << scale_path << std::endl;
|
| 286 |
+
return false;
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
file.read(reinterpret_cast<char*>(h_action_min_), 7 * sizeof(float));
|
| 290 |
+
file.read(reinterpret_cast<char*>(h_action_max_), 7 * sizeof(float));
|
| 291 |
+
|
| 292 |
+
cudaMemcpy(d_action_min_, h_action_min_, 7 * sizeof(float), cudaMemcpyHostToDevice);
|
| 293 |
+
cudaMemcpy(d_action_max_, h_action_max_, 7 * sizeof(float), cudaMemcpyHostToDevice);
|
| 294 |
+
return true;
|
| 295 |
+
}
|
| 296 |
+
|
| 297 |
+
float* ParvulusEngine::get_tensor(const std::string& name) {
|
| 298 |
+
auto it = tensors_.find(name);
|
| 299 |
+
if (it == tensors_.end()) {
|
| 300 |
+
std::cerr << "[ParvulusEngine] Error: Tensor not found: " << name << std::endl;
|
| 301 |
+
return nullptr;
|
| 302 |
+
}
|
| 303 |
+
return it->second.d_ptr;
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
// -------------------------------------------------------------
|
| 307 |
+
// Precompute Visual Key/Value Cache (Evaluated ONCE per chunk)
|
| 308 |
+
// -------------------------------------------------------------
|
| 309 |
+
void ParvulusEngine::precompute_kv_cache() {
|
| 310 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 311 |
+
for (int layer = 0; layer < 4; ++layer) {
|
| 312 |
+
std::string prefix = "action_decoder.layers." + std::to_string(layer) + ".";
|
| 313 |
+
float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight");
|
| 314 |
+
float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias");
|
| 315 |
+
if (!w_c_qkv) continue;
|
| 316 |
+
|
| 317 |
+
// In PyTorch MultiheadAttention in_proj_weight: shape [768, 256]
|
| 318 |
+
// K weight: rows 256..511 (offset 256 * 256), bias: rows 256..511 (offset 256)
|
| 319 |
+
// V weight: rows 512..767 (offset 512 * 256), bias: rows 512..767 (offset 512)
|
| 320 |
+
float* w_k = w_c_qkv + 256 * 256;
|
| 321 |
+
float* b_k = b_c_qkv + 256;
|
| 322 |
+
float* w_v = w_c_qkv + 512 * 256;
|
| 323 |
+
float* b_v = b_c_qkv + 512;
|
| 324 |
+
|
| 325 |
+
float* out_k = d_cached_k_ + layer * 8 * 16 * 32;
|
| 326 |
+
float* out_v = d_cached_v_ + layer * 8 * 16 * 32;
|
| 327 |
+
|
| 328 |
+
// 1. K = z_vis * W_K^T + b_K: [16, 256] x [256, 256]^T -> [16, 256]
|
| 329 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_k, 256, d_z_vis_, 256, &beta, d_temp_k_, 256);
|
| 330 |
+
add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_k_, b_k, 16, 256);
|
| 331 |
+
|
| 332 |
+
// 2. V = z_vis * W_V^T + b_V: [16, 256] x [256, 256]^T -> [16, 256]
|
| 333 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 16, 256, &alpha, w_v, 256, d_z_vis_, 256, &beta, d_temp_v_, 256);
|
| 334 |
+
add_bias_across_tokens_kernel<<<(16 * 256 + 255) / 256, 256, 0, stream_>>>(d_temp_v_, b_v, 16, 256);
|
| 335 |
+
|
| 336 |
+
// 3. Transpose to head-major: [16 tokens, 8 heads, 32 dims] -> [8 heads, 16 tokens, 32 dims]
|
| 337 |
+
transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
|
| 338 |
+
d_temp_k_, out_k, 16, 8, 32
|
| 339 |
+
);
|
| 340 |
+
transpose_interleaved_to_head_kernel<<<(16 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
|
| 341 |
+
d_temp_v_, out_v, 16, 8, 32
|
| 342 |
+
);
|
| 343 |
+
}
|
| 344 |
+
}
|
| 345 |
+
|
| 346 |
+
// -------------------------------------------------------------
|
| 347 |
+
// Timestep Conditioning MLP
|
| 348 |
+
// -------------------------------------------------------------
|
| 349 |
+
void ParvulusEngine::forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond) {
|
| 350 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 351 |
+
sinusoidal_embedding_kernel<<<1, 128, 0, stream_>>>(t, d_t_emb_, 256);
|
| 352 |
+
|
| 353 |
+
float* w0 = get_tensor("action_t_mlp.mlp.0.weight");
|
| 354 |
+
float* b0 = get_tensor("action_t_mlp.mlp.0.bias");
|
| 355 |
+
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w0, 256, d_t_emb_, 1, &beta, d_trans_norm_, 1);
|
| 356 |
+
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, b0, 256);
|
| 357 |
+
silu_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_norm_, 256);
|
| 358 |
+
|
| 359 |
+
float* w2 = get_tensor("action_t_mlp.mlp.2.weight");
|
| 360 |
+
float* b2 = get_tensor("action_t_mlp.mlp.2.bias");
|
| 361 |
+
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 256, &alpha, w2, 256, d_trans_norm_, 1, &beta, d_t_emb_, 1);
|
| 362 |
+
add_bias_1d_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_t_emb_, b2, 256);
|
| 363 |
+
|
| 364 |
+
add_residual_kernel<<<(256 + 255) / 256, 256, 0, stream_>>>(d_text_emb, d_t_emb_, d_cond, 256);
|
| 365 |
+
}
|
| 366 |
+
|
| 367 |
+
// -------------------------------------------------------------
|
| 368 |
+
// Fast KV-Cached Transformer Decoder Step
|
| 369 |
+
// -------------------------------------------------------------
|
| 370 |
+
void ParvulusEngine::forward_decoder_kv(
|
| 371 |
+
const float* d_z_sigma_8x7,
|
| 372 |
+
const float* d_cond_256,
|
| 373 |
+
float* d_v_pred_8x7
|
| 374 |
+
) {
|
| 375 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 376 |
+
|
| 377 |
+
// 1. Initial linear projection + pos_emb: h = z_sigma * W_in^T + b_in + pos_emb
|
| 378 |
+
float* w_in = get_tensor("action_decoder.action_in_proj.weight");
|
| 379 |
+
float* b_in = get_tensor("action_decoder.action_in_proj.bias");
|
| 380 |
+
float* w_pos = get_tensor("action_decoder.pos_emb");
|
| 381 |
+
|
| 382 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 7, &alpha, w_in, 7, d_z_sigma_8x7, 7, &beta, d_trans_h_, 256);
|
| 383 |
+
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, b_in, 8, 256);
|
| 384 |
+
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, w_pos, d_trans_h_, 8 * 256);
|
| 385 |
+
|
| 386 |
+
float scale = 0.1767766952966369f; // 1.0 / sqrt(32)
|
| 387 |
+
|
| 388 |
+
// 2. Loop over 4 Transformer Decoder Layers
|
| 389 |
+
for (int layer = 0; layer < 4; ++layer) {
|
| 390 |
+
std::string prefix = "action_decoder.layers." + std::to_string(layer) + ".";
|
| 391 |
+
|
| 392 |
+
// -------------------------------------------------------------
|
| 393 |
+
// (a) Temporal Self-Attention
|
| 394 |
+
// -------------------------------------------------------------
|
| 395 |
+
float* g_n1 = get_tensor(prefix + "norm1.weight");
|
| 396 |
+
float* b_n1 = get_tensor(prefix + "norm1.bias");
|
| 397 |
+
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n1, b_n1, d_trans_norm_, 8);
|
| 398 |
+
|
| 399 |
+
float* w_s_qkv = get_tensor(prefix + "self_attn.in_proj_weight");
|
| 400 |
+
float* b_s_qkv = get_tensor(prefix + "self_attn.in_proj_bias");
|
| 401 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 768, 8, 256, &alpha, w_s_qkv, 256, d_trans_norm_, 256, &beta, d_trans_qkv_, 768);
|
| 402 |
+
add_bias_across_tokens_kernel<<<(8 * 768 + 255) / 256, 256, 0, stream_>>>(d_trans_qkv_, b_s_qkv, 8, 768);
|
| 403 |
+
|
| 404 |
+
// Transpose QKV to head-major [8 heads, 8 tokens, 32 dims]
|
| 405 |
+
transpose_qkv_to_heads_kernel<<<8, 256, 0, stream_>>>(
|
| 406 |
+
d_trans_qkv_, d_trans_q_head_, d_trans_k_head_, d_trans_v_head_, 8, 8, 32
|
| 407 |
+
);
|
| 408 |
+
|
| 409 |
+
// Scores = Q_h * K_h^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 8 keys]
|
| 410 |
+
cublasSgemmStridedBatched(
|
| 411 |
+
cublas_handle_,
|
| 412 |
+
CUBLAS_OP_T, CUBLAS_OP_N,
|
| 413 |
+
8, 8, 32,
|
| 414 |
+
&scale,
|
| 415 |
+
d_trans_k_head_, 32, 256,
|
| 416 |
+
d_trans_q_head_, 32, 256,
|
| 417 |
+
&beta,
|
| 418 |
+
d_trans_scores_, 8, 64,
|
| 419 |
+
8
|
| 420 |
+
);
|
| 421 |
+
|
| 422 |
+
// Softmax across 8 keys
|
| 423 |
+
softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 8);
|
| 424 |
+
|
| 425 |
+
// Context = Scores * V_h -> [8 heads, 8 queries, 32 dims]
|
| 426 |
+
cublasSgemmStridedBatched(
|
| 427 |
+
cublas_handle_,
|
| 428 |
+
CUBLAS_OP_N, CUBLAS_OP_N,
|
| 429 |
+
32, 8, 8,
|
| 430 |
+
&alpha,
|
| 431 |
+
d_trans_v_head_, 32, 256,
|
| 432 |
+
d_trans_scores_, 8, 64,
|
| 433 |
+
&beta,
|
| 434 |
+
d_trans_ctx_head_, 32, 256,
|
| 435 |
+
8
|
| 436 |
+
);
|
| 437 |
+
|
| 438 |
+
// Transpose context to interleaved [8 tokens, 256 dims]
|
| 439 |
+
transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
|
| 440 |
+
d_trans_ctx_head_, d_trans_context_, 8, 8, 32
|
| 441 |
+
);
|
| 442 |
+
|
| 443 |
+
// Out projection & residual
|
| 444 |
+
float* w_s_out = get_tensor(prefix + "self_attn.out_proj.weight");
|
| 445 |
+
float* b_s_out = get_tensor(prefix + "self_attn.out_proj.bias");
|
| 446 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_s_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256);
|
| 447 |
+
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_s_out, 8, 256);
|
| 448 |
+
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256);
|
| 449 |
+
|
| 450 |
+
// -------------------------------------------------------------
|
| 451 |
+
// (b) Visual Cross-Attention with Cached K and V
|
| 452 |
+
// -------------------------------------------------------------
|
| 453 |
+
float* g_n2 = get_tensor(prefix + "norm2.weight");
|
| 454 |
+
float* b_n2 = get_tensor(prefix + "norm2.bias");
|
| 455 |
+
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n2, b_n2, d_trans_norm_, 8);
|
| 456 |
+
|
| 457 |
+
float* w_c_qkv = get_tensor(prefix + "cross_attn.in_proj_weight");
|
| 458 |
+
float* b_c_qkv = get_tensor(prefix + "cross_attn.in_proj_bias");
|
| 459 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_qkv, 256, d_trans_norm_, 256, &beta, d_q_proj_, 256);
|
| 460 |
+
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_q_proj_, b_c_qkv, 8, 256);
|
| 461 |
+
|
| 462 |
+
// Transpose Q to head-major [8 heads, 8 queries, 32 dims]
|
| 463 |
+
transpose_interleaved_to_head_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
|
| 464 |
+
d_q_proj_, d_trans_q_head_, 8, 8, 32
|
| 465 |
+
);
|
| 466 |
+
|
| 467 |
+
float* cached_k = d_cached_k_ + layer * 8 * 16 * 32;
|
| 468 |
+
float* cached_v = d_cached_v_ + layer * 8 * 16 * 32;
|
| 469 |
+
|
| 470 |
+
// Scores = Q_h * K_cached^T * (1 / sqrt(32)) -> [8 heads, 8 queries, 16 keys]
|
| 471 |
+
cublasSgemmStridedBatched(
|
| 472 |
+
cublas_handle_,
|
| 473 |
+
CUBLAS_OP_T, CUBLAS_OP_N,
|
| 474 |
+
16, 8, 32,
|
| 475 |
+
&scale,
|
| 476 |
+
cached_k, 32, 512,
|
| 477 |
+
d_trans_q_head_, 32, 256,
|
| 478 |
+
&beta,
|
| 479 |
+
d_trans_scores_, 16, 128,
|
| 480 |
+
8
|
| 481 |
+
);
|
| 482 |
+
|
| 483 |
+
// Softmax across 16 keys
|
| 484 |
+
softmax_small_kernel<<<8 * 8, 32, 0, stream_>>>(d_trans_scores_, 8 * 8, 16);
|
| 485 |
+
|
| 486 |
+
// Context = Scores * V_cached -> [8 heads, 8 queries, 32 dims]
|
| 487 |
+
cublasSgemmStridedBatched(
|
| 488 |
+
cublas_handle_,
|
| 489 |
+
CUBLAS_OP_N, CUBLAS_OP_N,
|
| 490 |
+
32, 8, 16,
|
| 491 |
+
&alpha,
|
| 492 |
+
cached_v, 32, 512,
|
| 493 |
+
d_trans_scores_, 16, 128,
|
| 494 |
+
&beta,
|
| 495 |
+
d_trans_ctx_head_, 32, 256,
|
| 496 |
+
8
|
| 497 |
+
);
|
| 498 |
+
|
| 499 |
+
// Transpose context to interleaved [8 tokens, 256 dims]
|
| 500 |
+
transpose_head_to_interleaved_kernel<<<(8 * 8 * 32 + 255) / 256, 256, 0, stream_>>>(
|
| 501 |
+
d_trans_ctx_head_, d_trans_context_, 8, 8, 32
|
| 502 |
+
);
|
| 503 |
+
|
| 504 |
+
// Out projection & residual
|
| 505 |
+
float* w_c_out = get_tensor(prefix + "cross_attn.out_proj.weight");
|
| 506 |
+
float* b_c_out = get_tensor(prefix + "cross_attn.out_proj.bias");
|
| 507 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 256, &alpha, w_c_out, 256, d_trans_context_, 256, &beta, d_trans_out_, 256);
|
| 508 |
+
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_c_out, 8, 256);
|
| 509 |
+
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256);
|
| 510 |
+
|
| 511 |
+
// -------------------------------------------------------------
|
| 512 |
+
// (c) FiLM Modulation & MLP
|
| 513 |
+
// -------------------------------------------------------------
|
| 514 |
+
float* w_film = get_tensor(prefix + "film.weight");
|
| 515 |
+
float* b_film = get_tensor(prefix + "film.bias");
|
| 516 |
+
cublasSgemv(cublas_handle_, CUBLAS_OP_T, 256, 512, &alpha, w_film, 256, d_cond_256, 1, &beta, d_trans_film_, 1);
|
| 517 |
+
add_bias_1d_kernel<<<(512 + 255) / 256, 256, 0, stream_>>>(d_trans_film_, b_film, 512);
|
| 518 |
+
|
| 519 |
+
float* g_n3 = get_tensor(prefix + "norm3.weight");
|
| 520 |
+
float* b_n3 = get_tensor(prefix + "norm3.bias");
|
| 521 |
+
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_n3, b_n3, d_trans_norm_, 8);
|
| 522 |
+
film_modulate_256_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_norm_, d_trans_film_, d_trans_norm_, 8);
|
| 523 |
+
|
| 524 |
+
float* w_m1 = get_tensor(prefix + "mlp.0.weight");
|
| 525 |
+
float* b_m1 = get_tensor(prefix + "mlp.0.bias");
|
| 526 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 1024, 8, 256, &alpha, w_m1, 256, d_trans_norm_, 256, &beta, d_trans_mlp_, 1024);
|
| 527 |
+
add_bias_across_tokens_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, b_m1, 8, 1024);
|
| 528 |
+
gelu_kernel<<<(8 * 1024 + 255) / 256, 256, 0, stream_>>>(d_trans_mlp_, d_trans_mlp_, 8 * 1024);
|
| 529 |
+
|
| 530 |
+
float* w_m2 = get_tensor(prefix + "mlp.2.weight");
|
| 531 |
+
float* b_m2 = get_tensor(prefix + "mlp.2.bias");
|
| 532 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 256, 8, 1024, &alpha, w_m2, 1024, d_trans_mlp_, 1024, &beta, d_trans_out_, 256);
|
| 533 |
+
add_bias_across_tokens_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_out_, b_m2, 8, 256);
|
| 534 |
+
add_residual_kernel<<<(8 * 256 + 255) / 256, 256, 0, stream_>>>(d_trans_h_, d_trans_out_, d_trans_h_, 8 * 256);
|
| 535 |
+
}
|
| 536 |
+
|
| 537 |
+
// 3. Final LayerNorm & Out Head (256 -> 7)
|
| 538 |
+
float* g_fn = get_tensor("action_decoder.final_norm.weight");
|
| 539 |
+
float* b_fn = get_tensor("action_decoder.final_norm.bias");
|
| 540 |
+
layernorm_256_kernel<<<8, 256, 0, stream_>>>(d_trans_h_, g_fn, b_fn, d_trans_norm_, 8);
|
| 541 |
+
|
| 542 |
+
float* w_out = get_tensor("action_decoder.out_head.weight");
|
| 543 |
+
float* b_out = get_tensor("action_decoder.out_head.bias");
|
| 544 |
+
cublasSgemm(cublas_handle_, CUBLAS_OP_T, CUBLAS_OP_N, 7, 8, 256, &alpha, w_out, 256, d_trans_norm_, 256, &beta, d_v_pred_8x7, 7);
|
| 545 |
+
add_bias_across_tokens_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_v_pred_8x7, b_out, 8, 7);
|
| 546 |
+
}
|
| 547 |
+
|
| 548 |
+
// -------------------------------------------------------------
|
| 549 |
+
// Vision DualViewEncoder Forward Pass
|
| 550 |
+
// -------------------------------------------------------------
|
| 551 |
+
void ParvulusEngine::forward_encoder(
|
| 552 |
+
const float* d_obs_6x256x256,
|
| 553 |
+
const float* d_text_emb_256,
|
| 554 |
+
float* d_z_vis_16x256
|
| 555 |
+
) {
|
| 556 |
+
dual_view_encoder_forward(
|
| 557 |
+
this,
|
| 558 |
+
cublas_handle_,
|
| 559 |
+
stream_,
|
| 560 |
+
d_obs_6x256x256,
|
| 561 |
+
d_text_emb_256,
|
| 562 |
+
d_z_vis_16x256,
|
| 563 |
+
d_vis_scratch_a_,
|
| 564 |
+
d_vis_scratch_b_,
|
| 565 |
+
d_vis_scratch_c_,
|
| 566 |
+
d_vis_scratch_d_,
|
| 567 |
+
d_im2col_,
|
| 568 |
+
d_film_ss_,
|
| 569 |
+
d_pool_k_,
|
| 570 |
+
d_pool_v_,
|
| 571 |
+
d_pool_k_head_,
|
| 572 |
+
d_pool_v_head_,
|
| 573 |
+
d_pool_q_,
|
| 574 |
+
d_pool_q_head_,
|
| 575 |
+
d_pool_scores_,
|
| 576 |
+
d_pool_ctx_head_,
|
| 577 |
+
d_pool_context_
|
| 578 |
+
);
|
| 579 |
+
}
|
| 580 |
+
|
| 581 |
+
// -------------------------------------------------------------
|
| 582 |
+
bool ParvulusEngine::register_task_embedding(const std::string& task_description, const float* emb_256) {
|
| 583 |
+
if (!emb_256) return false;
|
| 584 |
+
task_embeddings_[task_description] = std::vector<float>(emb_256, emb_256 + 256);
|
| 585 |
+
return true;
|
| 586 |
+
}
|
| 587 |
+
|
| 588 |
+
// -------------------------------------------------------------
|
| 589 |
+
// Complete Euler Action Flow Prediction Pipeline (Vector-First)
|
| 590 |
+
// -------------------------------------------------------------
|
| 591 |
+
bool ParvulusEngine::set_task_vector(const float* h_task_vector_256) {
|
| 592 |
+
if (!h_task_vector_256) return false;
|
| 593 |
+
cudaMemcpyAsync(d_text_emb_, h_task_vector_256, 256 * sizeof(float), cudaMemcpyHostToDevice, stream_);
|
| 594 |
+
has_task_vector_ = true;
|
| 595 |
+
return true;
|
| 596 |
+
}
|
| 597 |
+
|
| 598 |
+
bool ParvulusEngine::set_task_string(const std::string& task_description) {
|
| 599 |
+
auto it = task_embeddings_.find(task_description);
|
| 600 |
+
if (it == task_embeddings_.end()) {
|
| 601 |
+
return false;
|
| 602 |
+
}
|
| 603 |
+
return set_task_vector(it->second.data());
|
| 604 |
+
}
|
| 605 |
+
|
| 606 |
+
bool ParvulusEngine::predict_action_chunk(
|
| 607 |
+
const uint8_t* h_agent_img,
|
| 608 |
+
const uint8_t* h_wrist_img,
|
| 609 |
+
float* h_out_actions,
|
| 610 |
+
int num_euler_steps,
|
| 611 |
+
const float* custom_z0
|
| 612 |
+
) {
|
| 613 |
+
if (!has_task_vector_) {
|
| 614 |
+
std::cerr << "[ParvulusEngine] Error: Task conditioning vector has not been set! Call set_task_vector() first." << std::endl;
|
| 615 |
+
return false;
|
| 616 |
+
}
|
| 617 |
+
if (!h_agent_img || !h_wrist_img || !h_out_actions) {
|
| 618 |
+
return false;
|
| 619 |
+
}
|
| 620 |
+
|
| 621 |
+
// 1. Preprocess raw RGB frames into normalized [6, 256, 256] tensor
|
| 622 |
+
cudaMemcpyAsync(d_raw_agent_, h_agent_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_);
|
| 623 |
+
cudaMemcpyAsync(d_raw_wrist_, h_wrist_img, 256 * 256 * 3, cudaMemcpyHostToDevice, stream_);
|
| 624 |
+
|
| 625 |
+
preprocess_frames_kernel<<<(256 * 256 + 255) / 256, 256, 0, stream_>>>(
|
| 626 |
+
d_raw_agent_, d_raw_wrist_, d_obs_, 256, 256
|
| 627 |
+
);
|
| 628 |
+
|
| 629 |
+
// 2. Vision Forward Pass (Evaluated ONCE per chunk)
|
| 630 |
+
forward_encoder(d_obs_, d_text_emb_, d_z_vis_);
|
| 631 |
+
|
| 632 |
+
// 3. Precompute Visual KV Cache (Evaluated ONCE per chunk)
|
| 633 |
+
precompute_kv_cache();
|
| 634 |
+
|
| 635 |
+
// 4. Sample base noise z0
|
| 636 |
+
if (custom_z0 != nullptr) {
|
| 637 |
+
cudaMemcpyAsync(d_z_sigma_, custom_z0, 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_);
|
| 638 |
+
} else {
|
| 639 |
+
std::vector<float> h_z0(8 * 7);
|
| 640 |
+
for (int i = 0; i < 8 * 7; ++i) h_z0[i] = norm_dist_(rng_);
|
| 641 |
+
cudaMemcpyAsync(d_z_sigma_, h_z0.data(), 8 * 7 * sizeof(float), cudaMemcpyHostToDevice, stream_);
|
| 642 |
+
}
|
| 643 |
+
|
| 644 |
+
// 5. Euler ODE Integration Loop with KV Cache
|
| 645 |
+
float dt = 1.0f / (float)num_euler_steps;
|
| 646 |
+
for (int step = 0; step < num_euler_steps; ++step) {
|
| 647 |
+
float t = (float)step * dt;
|
| 648 |
+
forward_timestep_mlp(t, d_text_emb_, d_cond_);
|
| 649 |
+
forward_decoder_kv(d_z_sigma_, d_cond_, d_v_pred_);
|
| 650 |
+
euler_step_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, d_v_pred_, dt, 8 * 7);
|
| 651 |
+
}
|
| 652 |
+
clamp_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(d_z_sigma_, -1.0f, 1.0f, 8 * 7);
|
| 653 |
+
|
| 654 |
+
// 6. Denormalize actions to unnormalized robot space
|
| 655 |
+
denormalize_actions_kernel<<<(8 * 7 + 255) / 256, 256, 0, stream_>>>(
|
| 656 |
+
d_z_sigma_, d_action_min_, d_action_max_, d_final_actions_, 8, 7
|
| 657 |
+
);
|
| 658 |
+
|
| 659 |
+
cudaMemcpyAsync(h_out_actions, d_final_actions_, 8 * 7 * sizeof(float), cudaMemcpyDeviceToHost, stream_);
|
| 660 |
+
cudaStreamSynchronize(stream_);
|
| 661 |
+
return true;
|
| 662 |
+
}
|
| 663 |
+
|
| 664 |
+
bool ParvulusEngine::predict_action_chunk_with_vector(
|
| 665 |
+
const uint8_t* h_agent_img,
|
| 666 |
+
const uint8_t* h_wrist_img,
|
| 667 |
+
const float* h_task_vector_256,
|
| 668 |
+
float* h_out_actions,
|
| 669 |
+
int num_euler_steps,
|
| 670 |
+
const float* custom_z0
|
| 671 |
+
) {
|
| 672 |
+
if (!set_task_vector(h_task_vector_256)) return false;
|
| 673 |
+
return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0);
|
| 674 |
+
}
|
| 675 |
+
|
| 676 |
+
bool ParvulusEngine::predict_action_chunk(
|
| 677 |
+
const uint8_t* h_agent_img,
|
| 678 |
+
const uint8_t* h_wrist_img,
|
| 679 |
+
const std::string& task_description,
|
| 680 |
+
float* h_out_actions,
|
| 681 |
+
int num_euler_steps,
|
| 682 |
+
const float* custom_z0
|
| 683 |
+
) {
|
| 684 |
+
if (!set_task_string(task_description)) return false;
|
| 685 |
+
return predict_action_chunk(h_agent_img, h_wrist_img, h_out_actions, num_euler_steps, custom_z0);
|
| 686 |
+
}
|
c_engine/parvulus_engine.h
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <cuda_runtime.h>
|
| 4 |
+
#include <cublas_v2.h>
|
| 5 |
+
#include <string>
|
| 6 |
+
#include <vector>
|
| 7 |
+
#include <unordered_map>
|
| 8 |
+
#include <cstdint>
|
| 9 |
+
#include <random>
|
| 10 |
+
|
| 11 |
+
struct TensorInfo {
|
| 12 |
+
std::string name;
|
| 13 |
+
std::vector<int> shape;
|
| 14 |
+
size_t offset;
|
| 15 |
+
size_t size_bytes;
|
| 16 |
+
float* d_ptr = nullptr; // Device memory pointer
|
| 17 |
+
};
|
| 18 |
+
|
| 19 |
+
class ParvulusEngine {
|
| 20 |
+
public:
|
| 21 |
+
ParvulusEngine(int device_id = 0);
|
| 22 |
+
~ParvulusEngine();
|
| 23 |
+
|
| 24 |
+
bool load_weights(const std::string& weights_path);
|
| 25 |
+
bool load_task_embeddings(const std::string& embeddings_path);
|
| 26 |
+
bool load_action_scale(const std::string& scale_path);
|
| 27 |
+
bool register_task_embedding(const std::string& task_description, const float* emb_256);
|
| 28 |
+
|
| 29 |
+
// Sets the active task conditioning vector on GPU (256 floats).
|
| 30 |
+
// All subsequent chunk predictions will use this conditioning vector.
|
| 31 |
+
bool set_task_vector(const float* h_task_vector_256);
|
| 32 |
+
|
| 33 |
+
// Looks up a task string from cached dictionary and sets active conditioning vector.
|
| 34 |
+
bool set_task_string(const std::string& task_description);
|
| 35 |
+
|
| 36 |
+
// Predicts 8x7 action chunk using currently active task conditioning vector.
|
| 37 |
+
// agent_img, wrist_img: [256 * 256 * 3] uint8 in HWC format
|
| 38 |
+
// out_actions: float buffer of size [8 * 7] (56 floats)
|
| 39 |
+
bool predict_action_chunk(
|
| 40 |
+
const uint8_t* h_agent_img,
|
| 41 |
+
const uint8_t* h_wrist_img,
|
| 42 |
+
float* h_out_actions,
|
| 43 |
+
int num_euler_steps = 4,
|
| 44 |
+
const float* custom_z0 = nullptr
|
| 45 |
+
);
|
| 46 |
+
|
| 47 |
+
// Predicts 8x7 action chunk directly using a 256-dim task conditioning vector
|
| 48 |
+
bool predict_action_chunk_with_vector(
|
| 49 |
+
const uint8_t* h_agent_img,
|
| 50 |
+
const uint8_t* h_wrist_img,
|
| 51 |
+
const float* h_task_vector_256,
|
| 52 |
+
float* h_out_actions,
|
| 53 |
+
int num_euler_steps = 4,
|
| 54 |
+
const float* custom_z0 = nullptr
|
| 55 |
+
);
|
| 56 |
+
|
| 57 |
+
// Alias for predict_action_chunk_with_vector
|
| 58 |
+
bool predict_action_chunk_with_emb(
|
| 59 |
+
const uint8_t* h_agent_img,
|
| 60 |
+
const uint8_t* h_wrist_img,
|
| 61 |
+
const float* h_text_emb,
|
| 62 |
+
float* h_out_actions,
|
| 63 |
+
int num_euler_steps = 4,
|
| 64 |
+
const float* custom_z0 = nullptr
|
| 65 |
+
) {
|
| 66 |
+
return predict_action_chunk_with_vector(h_agent_img, h_wrist_img, h_text_emb, h_out_actions, num_euler_steps, custom_z0);
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
// Predicts 8x7 action chunk using a task string lookup
|
| 70 |
+
bool predict_action_chunk(
|
| 71 |
+
const uint8_t* h_agent_img,
|
| 72 |
+
const uint8_t* h_wrist_img,
|
| 73 |
+
const std::string& task_description,
|
| 74 |
+
float* h_out_actions,
|
| 75 |
+
int num_euler_steps = 4,
|
| 76 |
+
const float* custom_z0 = nullptr
|
| 77 |
+
);
|
| 78 |
+
|
| 79 |
+
// Direct tensor-to-tensor forward pass for parity verification
|
| 80 |
+
void forward_encoder(const float* d_obs_6x256x256, const float* d_text_emb_256, float* d_z_vis_16x256);
|
| 81 |
+
void forward_decoder(const float* d_z_sigma_8x7, const float* d_z_vis_16x256, const float* d_cond_256, float* d_v_pred_8x7);
|
| 82 |
+
void set_seed(unsigned int seed) { rng_.seed(seed); }
|
| 83 |
+
|
| 84 |
+
float* get_tensor(const std::string& name);
|
| 85 |
+
|
| 86 |
+
// Optimized Vision Encoder Buffers & Precomputed Weights
|
| 87 |
+
cudaStream_t get_stream() const { return stream_; }
|
| 88 |
+
float* get_gn_stats() const { return d_gn_stats_; }
|
| 89 |
+
float* get_attn_wk_fused(int s) const { return d_attn_pool_wk_fused_[s]; }
|
| 90 |
+
float* get_attn_bk_fused(int s) const { return d_attn_pool_bk_fused_[s]; }
|
| 91 |
+
float* get_attn_wv_fused(int s) const { return d_attn_pool_wv_fused_[s]; }
|
| 92 |
+
float* get_attn_bv_fused(int s) const { return d_attn_pool_bv_fused_[s]; }
|
| 93 |
+
float* get_attn_q_head(int s) const { return d_attn_pool_q_head_[s]; }
|
| 94 |
+
|
| 95 |
+
private:
|
| 96 |
+
int device_id_;
|
| 97 |
+
cublasHandle_t cublas_handle_;
|
| 98 |
+
cudaStream_t stream_;
|
| 99 |
+
std::mt19937 rng_;
|
| 100 |
+
std::normal_distribution<float> norm_dist_{0.0f, 1.0f};
|
| 101 |
+
|
| 102 |
+
// Weight storage
|
| 103 |
+
float* d_weights_buffer_ = nullptr;
|
| 104 |
+
size_t total_weight_bytes_ = 0;
|
| 105 |
+
std::unordered_map<std::string, TensorInfo> tensors_;
|
| 106 |
+
|
| 107 |
+
// Task text embeddings lookup
|
| 108 |
+
std::unordered_map<std::string, std::vector<float>> task_embeddings_;
|
| 109 |
+
float* d_text_emb_ = nullptr;
|
| 110 |
+
bool has_task_vector_ = false;
|
| 111 |
+
|
| 112 |
+
// Action scale bounds
|
| 113 |
+
float h_action_min_[7];
|
| 114 |
+
float h_action_max_[7];
|
| 115 |
+
float* d_action_min_ = nullptr;
|
| 116 |
+
float* d_action_max_ = nullptr;
|
| 117 |
+
|
| 118 |
+
// Static scratchpad device memory buffers (Zero allocations at runtime)
|
| 119 |
+
uint8_t* d_raw_agent_ = nullptr; // [256, 256, 3]
|
| 120 |
+
uint8_t* d_raw_wrist_ = nullptr; // [256, 256, 3]
|
| 121 |
+
float* d_final_actions_ = nullptr; // [8, 7]
|
| 122 |
+
float* d_obs_ = nullptr; // [6, 256, 256]
|
| 123 |
+
float* d_z_vis_ = nullptr; // [16, 256]
|
| 124 |
+
float* d_z_sigma_ = nullptr; // [8, 7]
|
| 125 |
+
float* d_v_pred_ = nullptr; // [8, 7]
|
| 126 |
+
float* d_cond_ = nullptr; // [256]
|
| 127 |
+
float* d_t_emb_ = nullptr; // [256]
|
| 128 |
+
|
| 129 |
+
// Intermediate vision activations scratchpad
|
| 130 |
+
float* d_vis_scratch_a_ = nullptr; // Large scratchpad A
|
| 131 |
+
float* d_vis_scratch_b_ = nullptr; // Large scratchpad B
|
| 132 |
+
float* d_vis_scratch_c_ = nullptr; // Large scratchpad C
|
| 133 |
+
float* d_vis_scratch_d_ = nullptr; // Large scratchpad D
|
| 134 |
+
float* d_im2col_ = nullptr; // Convolution im2col scratchpad
|
| 135 |
+
float* d_film_ss_ = nullptr; // FiLM scale/shift scratchpad
|
| 136 |
+
float* d_pool_k_ = nullptr; // AttnPool K scratchpad
|
| 137 |
+
float* d_pool_v_ = nullptr; // AttnPool V scratchpad
|
| 138 |
+
float* d_pool_k_head_ = nullptr; // AttnPool K head scratchpad
|
| 139 |
+
float* d_pool_v_head_ = nullptr; // AttnPool V head scratchpad
|
| 140 |
+
float* d_pool_q_ = nullptr; // AttnPool Q scratchpad
|
| 141 |
+
float* d_pool_q_head_ = nullptr; // AttnPool Q head scratchpad
|
| 142 |
+
float* d_pool_scores_ = nullptr; // AttnPool scores scratchpad
|
| 143 |
+
float* d_pool_ctx_head_ = nullptr; // AttnPool context head scratchpad
|
| 144 |
+
float* d_pool_context_ = nullptr; // AttnPool context scratchpad
|
| 145 |
+
float* d_gn_stats_ = nullptr; // GroupNorm multi-block atomic reduction scratchpad (16 floats)
|
| 146 |
+
|
| 147 |
+
// Precomputed / Fused AttnPool weights for stages 0..3 (96, 192, 192, 384 channels)
|
| 148 |
+
float* d_attn_pool_wk_fused_[4] = {nullptr};
|
| 149 |
+
float* d_attn_pool_bk_fused_[4] = {nullptr};
|
| 150 |
+
float* d_attn_pool_wv_fused_[4] = {nullptr};
|
| 151 |
+
float* d_attn_pool_bv_fused_[4] = {nullptr};
|
| 152 |
+
float* d_attn_pool_q_head_[4] = {nullptr};
|
| 153 |
+
|
| 154 |
+
// Transformer scratchpad
|
| 155 |
+
float* d_trans_h_ = nullptr; // [8, 256]
|
| 156 |
+
float* d_trans_norm_ = nullptr; // [8, 256]
|
| 157 |
+
float* d_trans_qkv_ = nullptr; // [8, 768]
|
| 158 |
+
float* d_trans_q_head_ = nullptr; // [8, 8, 32]
|
| 159 |
+
float* d_trans_k_head_ = nullptr; // [8, 16, 32]
|
| 160 |
+
float* d_trans_v_head_ = nullptr; // [8, 16, 32]
|
| 161 |
+
float* d_trans_scores_ = nullptr; // [8, 8, 16]
|
| 162 |
+
float* d_trans_ctx_head_ = nullptr; // [8, 8, 32]
|
| 163 |
+
float* d_trans_context_ = nullptr; // [8, 256]
|
| 164 |
+
float* d_trans_out_ = nullptr; // [8, 256]
|
| 165 |
+
float* d_trans_mlp_ = nullptr; // [8, 1024]
|
| 166 |
+
float* d_trans_film_ = nullptr; // [512]
|
| 167 |
+
float* d_temp_k_ = nullptr; // [16, 256]
|
| 168 |
+
float* d_temp_v_ = nullptr; // [16, 256]
|
| 169 |
+
|
| 170 |
+
// Precomputed Visual KV Cache buffers
|
| 171 |
+
float* d_cached_k_ = nullptr; // [4, 8, 16, 32]
|
| 172 |
+
float* d_cached_v_ = nullptr; // [4, 8, 16, 32]
|
| 173 |
+
float* d_q_proj_ = nullptr; // [8, 256]
|
| 174 |
+
|
| 175 |
+
// Native CUDA Graph capture for ODE loop
|
| 176 |
+
cudaGraph_t ode_graph_;
|
| 177 |
+
cudaGraphExec_t ode_graph_exec_ = nullptr;
|
| 178 |
+
bool graph_captured_ = false;
|
| 179 |
+
|
| 180 |
+
friend class DiagnosticRunner;
|
| 181 |
+
friend class Profiler;
|
| 182 |
+
void precompute_kv_cache();
|
| 183 |
+
void forward_timestep_mlp(float t, const float* d_text_emb, float* d_cond);
|
| 184 |
+
void forward_decoder_kv(const float* d_z_sigma_8x7, const float* d_cond_256, float* d_v_pred_8x7);
|
| 185 |
+
void allocate_scratchpads();
|
| 186 |
+
void free_scratchpads();
|
| 187 |
+
};
|
c_engine/parvulus_runtime_cli.cpp
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "parvulus_c_api.h"
|
| 2 |
+
|
| 3 |
+
#include <iostream>
|
| 4 |
+
#include <fstream>
|
| 5 |
+
#include <sstream>
|
| 6 |
+
#include <vector>
|
| 7 |
+
#include <string>
|
| 8 |
+
#include <chrono>
|
| 9 |
+
#include <iomanip>
|
| 10 |
+
#include <numeric>
|
| 11 |
+
#include <algorithm>
|
| 12 |
+
#include <cstring>
|
| 13 |
+
#include <cstdlib>
|
| 14 |
+
|
| 15 |
+
void print_help(const char* prog) {
|
| 16 |
+
std::cout << "====================================================================\n";
|
| 17 |
+
std::cout << " PARVULUS STANDALONE PURE CUDA RUNTIME CLI (Vector-First)\n";
|
| 18 |
+
std::cout << " Zero PyTorch, Zero LibTorch, Zero Python, Pure CUDA/cuBLAS Engine\n";
|
| 19 |
+
std::cout << "====================================================================\n\n";
|
| 20 |
+
std::cout << "Usage: " << prog << " [options]\n\n";
|
| 21 |
+
std::cout << "Options:\n";
|
| 22 |
+
std::cout << " --device <int> CUDA GPU device ID (default: 0)\n";
|
| 23 |
+
std::cout << " --weights <path> Path to parvulus_model_weights.bin\n";
|
| 24 |
+
std::cout << " --embeddings <path> Path to task_embeddings.bin (optional lookup table)\n";
|
| 25 |
+
std::cout << " --scale <path> Path to action_scale.bin (default: action_scale.bin)\n";
|
| 26 |
+
std::cout << " --nfe <int> Number of Euler ODE flow steps (default: 10)\n";
|
| 27 |
+
std::cout << "\nTask Conditioning (Vector-First):\n";
|
| 28 |
+
std::cout << " --task_vector_file <path> Load 256-dim task vector from file (.bin, .npy, .txt)\n";
|
| 29 |
+
std::cout << " --task_vector \"<f1,f2,...>\" Pass 256-dim float vector directly as string\n";
|
| 30 |
+
std::cout << " --task <str> Task instruction text (looks up in embeddings table)\n";
|
| 31 |
+
std::cout << "\nInputs & Outputs:\n";
|
| 32 |
+
std::cout << " --agent_img <path> Path to 256x256x3 uint8 raw RGB agentview image\n";
|
| 33 |
+
std::cout << " --wrist_img <path> Path to 256x256x3 uint8 raw RGB wrist image\n";
|
| 34 |
+
std::cout << " --output <path> File path to write predicted 8x7 action trajectory\n";
|
| 35 |
+
std::cout << " --benchmark <N> Run N chunk benchmark iterations and report latency\n";
|
| 36 |
+
std::cout << " -h, --help Show this help message and exit\n\n";
|
| 37 |
+
std::cout << "Examples:\n";
|
| 38 |
+
std::cout << " # 1. Run inference using a direct 256-dim task vector:\n";
|
| 39 |
+
std::cout << " " << prog << " --task_vector_file embedding.bin\n\n";
|
| 40 |
+
std::cout << " # 2. Run inference with a perturbed instruction vector or numpy array:\n";
|
| 41 |
+
std::cout << " " << prog << " --task_vector_file perturbed_emb.npy --output actions.txt\n\n";
|
| 42 |
+
std::cout << " # 3. High-throughput 100-chunk benchmark:\n";
|
| 43 |
+
std::cout << " " << prog << " --benchmark 100\n";
|
| 44 |
+
std::cout << "====================================================================\n";
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
bool file_exists(const std::string& path) {
|
| 48 |
+
std::ifstream f(path.c_str());
|
| 49 |
+
return f.good();
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
std::string find_file(const std::string& filename) {
|
| 53 |
+
if (file_exists(filename)) return filename;
|
| 54 |
+
if (file_exists("c_engine/" + filename)) return "c_engine/" + filename;
|
| 55 |
+
if (file_exists("../c_engine/" + filename)) return "../c_engine/" + filename;
|
| 56 |
+
return filename;
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
bool load_vector_from_file(const std::string& path, std::vector<float>& out_vec) {
|
| 60 |
+
out_vec.resize(256, 0.0f);
|
| 61 |
+
std::ifstream fb(path, std::ios::binary);
|
| 62 |
+
if (!fb.is_open()) return false;
|
| 63 |
+
|
| 64 |
+
fb.seekg(0, std::ios::end);
|
| 65 |
+
size_t sz = fb.tellg();
|
| 66 |
+
fb.seekg(0, std::ios::beg);
|
| 67 |
+
|
| 68 |
+
// 1. Raw float binary (256 * 4 = 1024 bytes)
|
| 69 |
+
if (sz == 256 * sizeof(float)) {
|
| 70 |
+
fb.read(reinterpret_cast<char*>(out_vec.data()), 256 * sizeof(float));
|
| 71 |
+
return true;
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
// 2. NumPy .npy format
|
| 75 |
+
if (sz >= 1024 && path.size() >= 4 && path.substr(path.size() - 4) == ".npy") {
|
| 76 |
+
fb.seekg(sz - 256 * sizeof(float), std::ios::beg);
|
| 77 |
+
fb.read(reinterpret_cast<char*>(out_vec.data()), 256 * sizeof(float));
|
| 78 |
+
return true;
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
// 3. ASCII whitespace or comma delimited text
|
| 82 |
+
fb.close();
|
| 83 |
+
std::ifstream ft(path);
|
| 84 |
+
std::string token;
|
| 85 |
+
int count = 0;
|
| 86 |
+
while (ft >> token && count < 256) {
|
| 87 |
+
if (!token.empty() && token.back() == ',') token.pop_back();
|
| 88 |
+
if (token.empty()) continue;
|
| 89 |
+
try {
|
| 90 |
+
out_vec[count++] = std::stof(token);
|
| 91 |
+
} catch (...) {}
|
| 92 |
+
}
|
| 93 |
+
return count == 256;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
bool parse_vector_from_string(const std::string& str, std::vector<float>& out_vec) {
|
| 97 |
+
out_vec.clear();
|
| 98 |
+
std::stringstream ss(str);
|
| 99 |
+
std::string token;
|
| 100 |
+
while (std::getline(ss, token, ',')) {
|
| 101 |
+
std::stringstream ss_tok(token);
|
| 102 |
+
float val;
|
| 103 |
+
while (ss_tok >> val) {
|
| 104 |
+
out_vec.push_back(val);
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
return out_vec.size() >= 256;
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
int main(int argc, char** argv) {
|
| 111 |
+
int device_id = 0;
|
| 112 |
+
int num_euler_steps = 10;
|
| 113 |
+
int benchmark_n = 0;
|
| 114 |
+
std::string weights_path = find_file("parvulus_model_weights.bin");
|
| 115 |
+
std::string embeddings_path = find_file("task_embeddings.bin");
|
| 116 |
+
std::string scale_path = find_file("action_scale.bin");
|
| 117 |
+
std::string task_desc = "pick up the black bowl between the plate and the ramekin and place it on the plate";
|
| 118 |
+
std::string task_vector_file = "";
|
| 119 |
+
std::string task_vector_str = "";
|
| 120 |
+
std::string agent_img_path = "";
|
| 121 |
+
std::string wrist_img_path = "";
|
| 122 |
+
std::string output_path = "";
|
| 123 |
+
|
| 124 |
+
for (int i = 1; i < argc; ++i) {
|
| 125 |
+
std::string arg = argv[i];
|
| 126 |
+
if (arg == "-h" || arg == "--help") {
|
| 127 |
+
print_help(argv[0]);
|
| 128 |
+
return 0;
|
| 129 |
+
} else if (arg == "--device" && i + 1 < argc) {
|
| 130 |
+
device_id = std::atoi(argv[++i]);
|
| 131 |
+
} else if (arg == "--weights" && i + 1 < argc) {
|
| 132 |
+
weights_path = argv[++i];
|
| 133 |
+
} else if (arg == "--embeddings" && i + 1 < argc) {
|
| 134 |
+
embeddings_path = argv[++i];
|
| 135 |
+
} else if (arg == "--scale" && i + 1 < argc) {
|
| 136 |
+
scale_path = argv[++i];
|
| 137 |
+
} else if (arg == "--nfe" && i + 1 < argc) {
|
| 138 |
+
num_euler_steps = std::atoi(argv[++i]);
|
| 139 |
+
} else if (arg == "--task" && i + 1 < argc) {
|
| 140 |
+
task_desc = argv[++i];
|
| 141 |
+
} else if ((arg == "--task_vector_file" || arg == "--task_emb_raw") && i + 1 < argc) {
|
| 142 |
+
task_vector_file = argv[++i];
|
| 143 |
+
} else if (arg == "--task_vector" && i + 1 < argc) {
|
| 144 |
+
task_vector_str = argv[++i];
|
| 145 |
+
} else if (arg == "--agent_img" && i + 1 < argc) {
|
| 146 |
+
agent_img_path = argv[++i];
|
| 147 |
+
} else if (arg == "--wrist_img" && i + 1 < argc) {
|
| 148 |
+
wrist_img_path = argv[++i];
|
| 149 |
+
} else if (arg == "--output" && i + 1 < argc) {
|
| 150 |
+
output_path = argv[++i];
|
| 151 |
+
} else if (arg == "--benchmark") {
|
| 152 |
+
benchmark_n = 100;
|
| 153 |
+
if (i + 1 < argc && argv[i + 1][0] != '-') {
|
| 154 |
+
benchmark_n = std::atoi(argv[++i]);
|
| 155 |
+
}
|
| 156 |
+
} else {
|
| 157 |
+
std::cerr << "Unknown option: " << arg << " (use --help for options)\n";
|
| 158 |
+
return 1;
|
| 159 |
+
}
|
| 160 |
+
}
|
| 161 |
+
|
| 162 |
+
std::cout << "====================================================================\n";
|
| 163 |
+
std::cout << " PARVULUS STANDALONE PURE CUDA RUNTIME (Vector-First)\n";
|
| 164 |
+
std::cout << " Zero PyTorch, Zero LibTorch, Standalone C/CUDA Engine\n";
|
| 165 |
+
std::cout << " CUDA Device: " << device_id << " | Euler ODE Steps: " << num_euler_steps << "\n";
|
| 166 |
+
std::cout << "====================================================================\n";
|
| 167 |
+
|
| 168 |
+
// 1. Instantiate pure CUDA engine
|
| 169 |
+
ParvulusHandle engine = parvulus_create(device_id);
|
| 170 |
+
if (!engine) {
|
| 171 |
+
std::cerr << "[Error] Failed to initialize CUDA engine on device " << device_id << "\n";
|
| 172 |
+
return 1;
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
// 2. Load model weights
|
| 176 |
+
std::cout << "[Init] Loading weights: " << weights_path << "...\n";
|
| 177 |
+
const char* emb_path_c = embeddings_path.empty() ? nullptr : embeddings_path.c_str();
|
| 178 |
+
const char* scl_path_c = scale_path.empty() ? nullptr : scale_path.c_str();
|
| 179 |
+
|
| 180 |
+
auto t_load_0 = std::chrono::high_resolution_clock::now();
|
| 181 |
+
bool ok = parvulus_load(engine, weights_path.c_str(), emb_path_c, scl_path_c);
|
| 182 |
+
auto t_load_1 = std::chrono::high_resolution_clock::now();
|
| 183 |
+
double load_ms = std::chrono::duration<double, std::milli>(t_load_1 - t_load_0).count();
|
| 184 |
+
|
| 185 |
+
if (!ok) {
|
| 186 |
+
std::cerr << "[Error] Failed to load binary model assets!\n";
|
| 187 |
+
parvulus_destroy(engine);
|
| 188 |
+
return 1;
|
| 189 |
+
}
|
| 190 |
+
std::cout << "[Init] Engine initialized in " << std::fixed << std::setprecision(2) << load_ms << " ms.\n";
|
| 191 |
+
|
| 192 |
+
// 3. Set Task Conditioning Vector (First-Class Vector Pipeline)
|
| 193 |
+
std::vector<float> task_vector(256, 0.0f);
|
| 194 |
+
bool vector_loaded = false;
|
| 195 |
+
|
| 196 |
+
if (!task_vector_file.empty()) {
|
| 197 |
+
if (load_vector_from_file(task_vector_file, task_vector)) {
|
| 198 |
+
vector_loaded = true;
|
| 199 |
+
std::cout << "[Task] Loaded 256-dim task conditioning vector from file: " << task_vector_file << "\n";
|
| 200 |
+
parvulus_set_task_vector(engine, task_vector.data());
|
| 201 |
+
} else {
|
| 202 |
+
std::cerr << "[Error] Failed to load 256 floats from: " << task_vector_file << "\n";
|
| 203 |
+
parvulus_destroy(engine);
|
| 204 |
+
return 1;
|
| 205 |
+
}
|
| 206 |
+
} else if (!task_vector_str.empty()) {
|
| 207 |
+
if (parse_vector_from_string(task_vector_str, task_vector)) {
|
| 208 |
+
vector_loaded = true;
|
| 209 |
+
std::cout << "[Task] Parsed 256-dim task conditioning vector from command-line string.\n";
|
| 210 |
+
parvulus_set_task_vector(engine, task_vector.data());
|
| 211 |
+
} else {
|
| 212 |
+
std::cerr << "[Error] Failed to parse 256 floats from --task_vector string!\n";
|
| 213 |
+
parvulus_destroy(engine);
|
| 214 |
+
return 1;
|
| 215 |
+
}
|
| 216 |
+
} else {
|
| 217 |
+
// Fall back to lookup string if provided
|
| 218 |
+
std::cout << "[Task] Instruction: \"" << task_desc << "\" (using lookup table)\n";
|
| 219 |
+
if (!parvulus_set_task_string(engine, task_desc.c_str())) {
|
| 220 |
+
std::cerr << "[Error] Task prompt not found in lookup cache: " << task_desc << "\n";
|
| 221 |
+
parvulus_destroy(engine);
|
| 222 |
+
return 1;
|
| 223 |
+
}
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
// 4. Prepare Image Buffers
|
| 227 |
+
const size_t img_size = 256 * 256 * 3;
|
| 228 |
+
std::vector<uint8_t> agent_img(img_size, 128);
|
| 229 |
+
std::vector<uint8_t> wrist_img(img_size, 128);
|
| 230 |
+
|
| 231 |
+
if (!agent_img_path.empty()) {
|
| 232 |
+
std::ifstream f(agent_img_path, std::ios::binary);
|
| 233 |
+
if (!f.is_open() || !f.read(reinterpret_cast<char*>(agent_img.data()), img_size)) {
|
| 234 |
+
std::cerr << "[Warning] Could not read agent image from: " << agent_img_path << ", using test pattern.\n";
|
| 235 |
+
} else {
|
| 236 |
+
std::cout << "[Input] Loaded agent image: " << agent_img_path << "\n";
|
| 237 |
+
}
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
if (!wrist_img_path.empty()) {
|
| 241 |
+
std::ifstream f(wrist_img_path, std::ios::binary);
|
| 242 |
+
if (!f.is_open() || !f.read(reinterpret_cast<char*>(wrist_img.data()), img_size)) {
|
| 243 |
+
std::cerr << "[Warning] Could not read wrist image from: " << wrist_img_path << ", using test pattern.\n";
|
| 244 |
+
} else {
|
| 245 |
+
std::cout << "[Input] Loaded wrist image: " << wrist_img_path << "\n";
|
| 246 |
+
}
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
std::vector<float> actions(8 * 7, 0.0f);
|
| 250 |
+
|
| 251 |
+
// 5. Benchmark or Single Prediction
|
| 252 |
+
if (benchmark_n > 0) {
|
| 253 |
+
std::cout << "\n[Benchmark] Warming up (10 iterations)...\n";
|
| 254 |
+
for (int i = 0; i < 10; ++i) {
|
| 255 |
+
parvulus_predict_chunk(engine, agent_img.data(), wrist_img.data(), nullptr, actions.data(), num_euler_steps, nullptr);
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
std::cout << "[Benchmark] Running " << benchmark_n << " consecutive chunk predictions...\n";
|
| 259 |
+
std::vector<double> latencies_ms;
|
| 260 |
+
latencies_ms.reserve(benchmark_n);
|
| 261 |
+
|
| 262 |
+
for (int i = 0; i < benchmark_n; ++i) {
|
| 263 |
+
auto t0 = std::chrono::high_resolution_clock::now();
|
| 264 |
+
parvulus_predict_chunk(engine, agent_img.data(), wrist_img.data(), nullptr, actions.data(), num_euler_steps, nullptr);
|
| 265 |
+
auto t1 = std::chrono::high_resolution_clock::now();
|
| 266 |
+
latencies_ms.push_back(std::chrono::duration<double, std::milli>(t1 - t0).count());
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
std::sort(latencies_ms.begin(), latencies_ms.end());
|
| 270 |
+
double min_ms = latencies_ms.front();
|
| 271 |
+
double max_ms = latencies_ms.back();
|
| 272 |
+
double mean_ms = std::accumulate(latencies_ms.begin(), latencies_ms.end(), 0.0) / benchmark_n;
|
| 273 |
+
double p50_ms = latencies_ms[benchmark_n / 2];
|
| 274 |
+
double p95_ms = latencies_ms[static_cast<size_t>(benchmark_n * 0.95)];
|
| 275 |
+
double p99_ms = latencies_ms[static_cast<size_t>(benchmark_n * 0.99)];
|
| 276 |
+
double action_rate = 8.0 / (mean_ms / 1000.0);
|
| 277 |
+
|
| 278 |
+
std::cout << "\n====================================================================\n";
|
| 279 |
+
std::cout << " LATENCY BENCHMARK RESULTS (N=" << benchmark_n << " chunks):\n";
|
| 280 |
+
std::cout << "====================================================================\n";
|
| 281 |
+
std::cout << " - Min Latency: " << std::fixed << std::setprecision(3) << min_ms << " ms\n";
|
| 282 |
+
std::cout << " - p50 Latency: " << std::fixed << std::setprecision(3) << p50_ms << " ms\n";
|
| 283 |
+
std::cout << " - Mean Chunk Latency: " << std::fixed << std::setprecision(3) << mean_ms << " ms\n";
|
| 284 |
+
std::cout << " - p95 Latency: " << std::fixed << std::setprecision(3) << p95_ms << " ms\n";
|
| 285 |
+
std::cout << " - p99 Latency: " << std::fixed << std::setprecision(3) << p99_ms << " ms\n";
|
| 286 |
+
std::cout << " - Max Latency: " << std::fixed << std::setprecision(3) << max_ms << " ms\n";
|
| 287 |
+
std::cout << " - Effective Action Rate: " << std::fixed << std::setprecision(1) << action_rate << " Hz\n";
|
| 288 |
+
std::cout << "====================================================================\n";
|
| 289 |
+
} else {
|
| 290 |
+
auto t0 = std::chrono::high_resolution_clock::now();
|
| 291 |
+
bool pred_ok = parvulus_predict_chunk(engine, agent_img.data(), wrist_img.data(), nullptr, actions.data(), num_euler_steps, nullptr);
|
| 292 |
+
auto t1 = std::chrono::high_resolution_clock::now();
|
| 293 |
+
double pred_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
|
| 294 |
+
|
| 295 |
+
if (!pred_ok) {
|
| 296 |
+
std::cerr << "[Error] Action chunk prediction failed!\n";
|
| 297 |
+
parvulus_destroy(engine);
|
| 298 |
+
return 1;
|
| 299 |
+
}
|
| 300 |
+
std::cout << "\n[Inference] Predicted 8x7 action chunk in " << std::fixed << std::setprecision(2) << pred_ms << " ms.\n";
|
| 301 |
+
}
|
| 302 |
+
|
| 303 |
+
// 6. Print Action Trajectory
|
| 304 |
+
std::cout << "\nPredicted Action Chunk [8 Steps x 7 Dimensions (dx, dy, dz, droll, dpitch, dyaw, gripper)]:\n";
|
| 305 |
+
std::cout << "--------------------------------------------------------------------------------------\n";
|
| 306 |
+
for (int step = 0; step < 8; ++step) {
|
| 307 |
+
std::cout << " Step " << step << ": [";
|
| 308 |
+
for (int d = 0; d < 7; ++d) {
|
| 309 |
+
std::cout << std::setw(8) << std::fixed << std::setprecision(4) << actions[step * 7 + d] << (d < 6 ? ", " : "");
|
| 310 |
+
}
|
| 311 |
+
std::cout << "]\n";
|
| 312 |
+
}
|
| 313 |
+
std::cout << "--------------------------------------------------------------------------------------\n";
|
| 314 |
+
|
| 315 |
+
// 7. Write to output file if requested
|
| 316 |
+
if (!output_path.empty()) {
|
| 317 |
+
std::ofstream out_f(output_path);
|
| 318 |
+
if (out_f.is_open()) {
|
| 319 |
+
for (int step = 0; step < 8; ++step) {
|
| 320 |
+
for (int d = 0; d < 7; ++d) {
|
| 321 |
+
out_f << std::fixed << std::setprecision(6) << actions[step * 7 + d] << (d < 6 ? " " : "\n");
|
| 322 |
+
}
|
| 323 |
+
}
|
| 324 |
+
std::cout << "[Output] Saved action chunk to: " << output_path << "\n";
|
| 325 |
+
} else {
|
| 326 |
+
std::cerr << "[Warning] Failed to write output to: " << output_path << "\n";
|
| 327 |
+
}
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
// 8. Cleanup
|
| 331 |
+
parvulus_destroy(engine);
|
| 332 |
+
std::cout << "\n[Pure CUDA] Engine released cleanly.\n";
|
| 333 |
+
return 0;
|
| 334 |
+
}
|
c_engine/test_vision_encoder.cpp
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "parvulus_engine.h"
|
| 2 |
+
#include "vision_encoder.h"
|
| 3 |
+
#include <iostream>
|
| 4 |
+
#include <fstream>
|
| 5 |
+
#include <vector>
|
| 6 |
+
#include <cmath>
|
| 7 |
+
#include <chrono>
|
| 8 |
+
#include <iomanip>
|
| 9 |
+
|
| 10 |
+
int main() {
|
| 11 |
+
std::cout << "=========================================================\n";
|
| 12 |
+
std::cout << " VISION ENCODER STANDALONE SELF-TEST & BENCHMARK\n";
|
| 13 |
+
std::cout << "=========================================================\n";
|
| 14 |
+
|
| 15 |
+
ParvulusEngine engine(0);
|
| 16 |
+
if (!engine.load_weights("parvulus_model_weights.bin")) {
|
| 17 |
+
std::cerr << "Failed to load parvulus_model_weights.bin\n";
|
| 18 |
+
return 1;
|
| 19 |
+
}
|
| 20 |
+
|
| 21 |
+
std::vector<float> h_obs(6 * 256 * 256, 0.5f);
|
| 22 |
+
std::vector<float> h_text(256, 0.1f);
|
| 23 |
+
|
| 24 |
+
float *d_obs, *d_text, *d_z_vis;
|
| 25 |
+
cudaMalloc(&d_obs, 6 * 256 * 256 * sizeof(float));
|
| 26 |
+
cudaMalloc(&d_text, 256 * sizeof(float));
|
| 27 |
+
cudaMalloc(&d_z_vis, 16 * 256 * sizeof(float));
|
| 28 |
+
|
| 29 |
+
cudaMemcpy(d_obs, h_obs.data(), h_obs.size() * sizeof(float), cudaMemcpyHostToDevice);
|
| 30 |
+
cudaMemcpy(d_text, h_text.data(), h_text.size() * sizeof(float), cudaMemcpyHostToDevice);
|
| 31 |
+
|
| 32 |
+
std::cout << "[Test] Warming up Vision Backbone (20 iterations)...\n";
|
| 33 |
+
for (int i = 0; i < 20; ++i) {
|
| 34 |
+
engine.forward_encoder(d_obs, d_text, d_z_vis);
|
| 35 |
+
}
|
| 36 |
+
cudaDeviceSynchronize();
|
| 37 |
+
|
| 38 |
+
std::cout << "[Test] Benchmarking 100 Vision Encoder iterations...\n";
|
| 39 |
+
auto t0 = std::chrono::high_resolution_clock::now();
|
| 40 |
+
for (int i = 0; i < 100; ++i) {
|
| 41 |
+
engine.forward_encoder(d_obs, d_text, d_z_vis);
|
| 42 |
+
}
|
| 43 |
+
cudaDeviceSynchronize();
|
| 44 |
+
auto t1 = std::chrono::high_resolution_clock::now();
|
| 45 |
+
|
| 46 |
+
double avg_ms = std::chrono::duration<double, std::milli>(t1 - t0).count() / 100.0;
|
| 47 |
+
std::cout << " -> Average Vision Encoder Latency: " << std::fixed << std::setprecision(3) << avg_ms << " ms\n";
|
| 48 |
+
|
| 49 |
+
std::vector<float> h_z_vis(16 * 256);
|
| 50 |
+
cudaMemcpy(h_z_vis.data(), d_z_vis, 16 * 256 * sizeof(float), cudaMemcpyDeviceToHost);
|
| 51 |
+
|
| 52 |
+
// Verify finite values and non-trivial output
|
| 53 |
+
bool valid = true;
|
| 54 |
+
for (float v : h_z_vis) {
|
| 55 |
+
if (std::isnan(v) || std::isinf(v)) {
|
| 56 |
+
valid = false;
|
| 57 |
+
break;
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
cudaFree(d_obs);
|
| 62 |
+
cudaFree(d_text);
|
| 63 |
+
cudaFree(d_z_vis);
|
| 64 |
+
|
| 65 |
+
if (valid) {
|
| 66 |
+
std::cout << " -> Vision output verified (finite, stable).\n";
|
| 67 |
+
std::cout << "=========================================================\n";
|
| 68 |
+
std::cout << " VISION ENCODER TEST PASSED!\n";
|
| 69 |
+
std::cout << "=========================================================\n";
|
| 70 |
+
return 0;
|
| 71 |
+
} else {
|
| 72 |
+
std::cerr << " -> Error: Output contains NaN or Inf!\n";
|
| 73 |
+
return 1;
|
| 74 |
+
}
|
| 75 |
+
}
|
c_engine/vision_encoder.cu
ADDED
|
@@ -0,0 +1,555 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "vision_encoder.cuh"
|
| 2 |
+
#include "kernels.cuh"
|
| 3 |
+
#include "parvulus_engine.h"
|
| 4 |
+
#include <iostream>
|
| 5 |
+
|
| 6 |
+
// -----------------------------------------------------------------------------
|
| 7 |
+
// Bias & Transposition Kernels
|
| 8 |
+
// -----------------------------------------------------------------------------
|
| 9 |
+
|
| 10 |
+
__global__ void add_bias_1d_kernel(float* data, const float* bias, int dim) {
|
| 11 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 12 |
+
if (idx < dim) {
|
| 13 |
+
data[idx] += bias[idx];
|
| 14 |
+
}
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
__global__ void add_bias_across_tokens_kernel(float* data, const float* bias, int num_tokens, int dim) {
|
| 18 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 19 |
+
int total = num_tokens * dim;
|
| 20 |
+
if (idx < total) {
|
| 21 |
+
int d = idx % dim;
|
| 22 |
+
data[idx] += bias[d];
|
| 23 |
+
}
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
__global__ void transpose_interleaved_to_head_kernel(
|
| 27 |
+
const float* __restrict__ in,
|
| 28 |
+
float* __restrict__ out,
|
| 29 |
+
int N, int H, int D
|
| 30 |
+
) {
|
| 31 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 32 |
+
int total = N * H * D;
|
| 33 |
+
if (idx >= total) return;
|
| 34 |
+
int d = idx % D;
|
| 35 |
+
int h = (idx / D) % H;
|
| 36 |
+
int n = idx / (H * D);
|
| 37 |
+
|
| 38 |
+
int in_idx = idx;
|
| 39 |
+
int out_idx = h * (N * D) + n * D + d;
|
| 40 |
+
out[out_idx] = in[in_idx];
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
__global__ void transpose_head_to_interleaved_kernel(
|
| 44 |
+
const float* __restrict__ in,
|
| 45 |
+
float* __restrict__ out,
|
| 46 |
+
int N, int H, int D
|
| 47 |
+
) {
|
| 48 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 49 |
+
int total = N * H * D;
|
| 50 |
+
if (idx >= total) return;
|
| 51 |
+
int d = idx % D;
|
| 52 |
+
int n = (idx / D) % N;
|
| 53 |
+
int h = idx / (N * D);
|
| 54 |
+
|
| 55 |
+
int out_idx = n * (H * D) + h * D + d;
|
| 56 |
+
out[out_idx] = in[idx];
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
// -----------------------------------------------------------------------------
|
| 60 |
+
// Convolution Primitives
|
| 61 |
+
// -----------------------------------------------------------------------------
|
| 62 |
+
|
| 63 |
+
void conv2d_3x3_forward(
|
| 64 |
+
cublasHandle_t handle,
|
| 65 |
+
cudaStream_t stream,
|
| 66 |
+
const float* d_in,
|
| 67 |
+
const float* d_weight,
|
| 68 |
+
const float* d_bias,
|
| 69 |
+
float* d_out,
|
| 70 |
+
float* d_im2col,
|
| 71 |
+
int in_c, int in_h, int in_w,
|
| 72 |
+
int out_c, int stride, int pad
|
| 73 |
+
) {
|
| 74 |
+
int out_h = (in_h + 2 * pad - 3) / stride + 1;
|
| 75 |
+
int out_w = (in_w + 2 * pad - 3) / stride + 1;
|
| 76 |
+
int total_spatial = out_h * out_w;
|
| 77 |
+
int total_elements = in_c * total_spatial;
|
| 78 |
+
|
| 79 |
+
int threads = 256;
|
| 80 |
+
int blocks = (total_elements + threads - 1) / threads;
|
| 81 |
+
im2col_3x3_kernel<<<blocks, threads, 0, stream>>>(
|
| 82 |
+
d_in, d_im2col, in_c, in_h, in_w, pad, stride, out_h, out_w
|
| 83 |
+
);
|
| 84 |
+
|
| 85 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 86 |
+
cublasSgemm(
|
| 87 |
+
handle,
|
| 88 |
+
CUBLAS_OP_N, CUBLAS_OP_N,
|
| 89 |
+
total_spatial, out_c, in_c * 9,
|
| 90 |
+
&alpha,
|
| 91 |
+
d_im2col, total_spatial,
|
| 92 |
+
d_weight, in_c * 9,
|
| 93 |
+
&beta,
|
| 94 |
+
d_out, total_spatial
|
| 95 |
+
);
|
| 96 |
+
|
| 97 |
+
if (d_bias) {
|
| 98 |
+
int total = out_c * total_spatial;
|
| 99 |
+
add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>(
|
| 100 |
+
d_out, d_bias, out_c, total_spatial
|
| 101 |
+
);
|
| 102 |
+
}
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
void conv2d_1x1_forward(
|
| 106 |
+
cublasHandle_t handle,
|
| 107 |
+
cudaStream_t stream,
|
| 108 |
+
const float* d_in,
|
| 109 |
+
const float* d_weight,
|
| 110 |
+
const float* d_bias,
|
| 111 |
+
float* d_out,
|
| 112 |
+
int in_c, int spatial, int out_c
|
| 113 |
+
) {
|
| 114 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 115 |
+
cublasSgemm(
|
| 116 |
+
handle,
|
| 117 |
+
CUBLAS_OP_N, CUBLAS_OP_N,
|
| 118 |
+
spatial, out_c, in_c,
|
| 119 |
+
&alpha,
|
| 120 |
+
d_in, spatial,
|
| 121 |
+
d_weight, in_c,
|
| 122 |
+
&beta,
|
| 123 |
+
d_out, spatial
|
| 124 |
+
);
|
| 125 |
+
|
| 126 |
+
if (d_bias) {
|
| 127 |
+
int total = out_c * spatial;
|
| 128 |
+
add_bias_2d_kernel<<<(total + 255) / 256, 256, 0, stream>>>(
|
| 129 |
+
d_out, d_bias, out_c, spatial
|
| 130 |
+
);
|
| 131 |
+
}
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
// -----------------------------------------------------------------------------
|
| 135 |
+
// FiLM ResBlock 2D Forward
|
| 136 |
+
// -----------------------------------------------------------------------------
|
| 137 |
+
|
| 138 |
+
void film_resblock_forward(
|
| 139 |
+
cublasHandle_t handle,
|
| 140 |
+
cudaStream_t stream,
|
| 141 |
+
const float* d_in,
|
| 142 |
+
float* d_out,
|
| 143 |
+
float* d_scratch1,
|
| 144 |
+
float* d_scratch2,
|
| 145 |
+
float* d_im2col,
|
| 146 |
+
float* d_film_ss,
|
| 147 |
+
const float* d_cond,
|
| 148 |
+
const std::string& prefix,
|
| 149 |
+
ParvulusEngine* engine,
|
| 150 |
+
int in_c, int out_c, int H, int W
|
| 151 |
+
) {
|
| 152 |
+
int spatial = H * W;
|
| 153 |
+
|
| 154 |
+
// 1. Fast GroupNorm + SiLU on d_in -> d_scratch1 (64-block parallel reduction)
|
| 155 |
+
float* g_n1 = engine->get_tensor(prefix + "norm1.weight");
|
| 156 |
+
float* b_n1 = engine->get_tensor(prefix + "norm1.bias");
|
| 157 |
+
cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream);
|
| 158 |
+
fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_in, engine->get_gn_stats(), in_c, spatial);
|
| 159 |
+
fast_groupnorm_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_in, engine->get_gn_stats(), g_n1, b_n1, d_scratch1, in_c, spatial, true);
|
| 160 |
+
|
| 161 |
+
// 2. conv1 (3x3, in_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2
|
| 162 |
+
float* w_c1 = engine->get_tensor(prefix + "conv1.weight");
|
| 163 |
+
float* b_c1 = engine->get_tensor(prefix + "conv1.bias");
|
| 164 |
+
conv2d_3x3_forward(handle, stream, d_scratch1, w_c1, b_c1, d_scratch2, d_im2col, in_c, H, W, out_c, 1, 1);
|
| 165 |
+
|
| 166 |
+
// 3. FiLM linear projection: Linear(256, 2 * out_c)
|
| 167 |
+
// d_film_ss = cond * W^T + b
|
| 168 |
+
float* w_film = engine->get_tensor(prefix + "film.weight");
|
| 169 |
+
float* b_film = engine->get_tensor(prefix + "film.bias");
|
| 170 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 171 |
+
cublasSgemv(handle, CUBLAS_OP_T, 256, 2 * out_c, &alpha, w_film, 256, d_cond, 1, &beta, d_film_ss, 1);
|
| 172 |
+
add_bias_1d_kernel<<<(2 * out_c + 255) / 256, 256, 0, stream>>>(d_film_ss, b_film, 2 * out_c);
|
| 173 |
+
|
| 174 |
+
// 4. Fast GroupNorm + FiLM modulation + SiLU on d_scratch2 -> d_scratch1 (64-block parallel reduction)
|
| 175 |
+
float* g_n2 = engine->get_tensor(prefix + "norm2.weight");
|
| 176 |
+
float* b_n2 = engine->get_tensor(prefix + "norm2.bias");
|
| 177 |
+
float* d_scale = d_film_ss;
|
| 178 |
+
float* d_shift = d_film_ss + out_c;
|
| 179 |
+
cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream);
|
| 180 |
+
fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch2, engine->get_gn_stats(), out_c, spatial);
|
| 181 |
+
fast_groupnorm_film_silu_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(
|
| 182 |
+
d_scratch2, engine->get_gn_stats(), g_n2, b_n2, d_scale, d_shift, d_scratch1, out_c, spatial
|
| 183 |
+
);
|
| 184 |
+
|
| 185 |
+
// 5. conv2 (3x3, out_c -> out_c, pad=1, stride=1) on d_scratch1 -> d_scratch2
|
| 186 |
+
float* w_c2 = engine->get_tensor(prefix + "conv2.weight");
|
| 187 |
+
float* b_c2 = engine->get_tensor(prefix + "conv2.bias");
|
| 188 |
+
conv2d_3x3_forward(handle, stream, d_scratch1, w_c2, b_c2, d_scratch2, d_im2col, out_c, H, W, out_c, 1, 1);
|
| 189 |
+
|
| 190 |
+
// 6. Skip connection
|
| 191 |
+
if (in_c != out_c) {
|
| 192 |
+
float* w_skip = engine->get_tensor(prefix + "skip.weight");
|
| 193 |
+
float* b_skip = engine->get_tensor(prefix + "skip.bias");
|
| 194 |
+
conv2d_1x1_forward(handle, stream, d_in, w_skip, b_skip, d_scratch1, in_c, spatial, out_c);
|
| 195 |
+
add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>(
|
| 196 |
+
d_scratch2, d_scratch1, d_out, out_c * spatial
|
| 197 |
+
);
|
| 198 |
+
} else {
|
| 199 |
+
add_residual_kernel<<<(out_c * spatial + 255) / 256, 256, 0, stream>>>(
|
| 200 |
+
d_scratch2, d_in, d_out, out_c * spatial
|
| 201 |
+
);
|
| 202 |
+
}
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
// -------------------------------------------------------------
|
| 206 |
+
// Cross-View Mixer Forward
|
| 207 |
+
// -------------------------------------------------------------
|
| 208 |
+
|
| 209 |
+
void cv_mixer_forward(
|
| 210 |
+
cublasHandle_t handle,
|
| 211 |
+
cudaStream_t stream,
|
| 212 |
+
const float* d_agent,
|
| 213 |
+
const float* d_wrist,
|
| 214 |
+
float* d_agent_out,
|
| 215 |
+
float* d_wrist_out,
|
| 216 |
+
float* d_mixed_out,
|
| 217 |
+
float* d_scratch1,
|
| 218 |
+
float* d_scratch2,
|
| 219 |
+
float* d_im2col,
|
| 220 |
+
const std::string& prefix,
|
| 221 |
+
ParvulusEngine* engine,
|
| 222 |
+
int ch, int H, int W
|
| 223 |
+
) {
|
| 224 |
+
int spatial = H * W;
|
| 225 |
+
|
| 226 |
+
// 1. Joint = cat(d_agent, d_wrist) -> shape [2*ch, spatial] in d_mixed_out
|
| 227 |
+
cudaMemcpyAsync(d_mixed_out, d_agent, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
|
| 228 |
+
cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
|
| 229 |
+
|
| 230 |
+
// 2. context_conv: Conv1x1 (2*ch -> ch) on joint -> GroupNorm(8, ch) -> SiLU -> ctx
|
| 231 |
+
float* w_ctx = engine->get_tensor(prefix + "context_conv.0.weight");
|
| 232 |
+
float* b_ctx = engine->get_tensor(prefix + "context_conv.0.bias");
|
| 233 |
+
conv2d_1x1_forward(handle, stream, d_mixed_out, w_ctx, b_ctx, d_scratch1, 2 * ch, spatial, ch);
|
| 234 |
+
|
| 235 |
+
float* g_ctx = engine->get_tensor(prefix + "context_conv.1.weight");
|
| 236 |
+
float* b_ctx_gn = engine->get_tensor(prefix + "context_conv.1.bias");
|
| 237 |
+
float* d_ctx = d_scratch2;
|
| 238 |
+
cudaMemsetAsync(engine->get_gn_stats(), 0, 16 * sizeof(float), stream);
|
| 239 |
+
fast_groupnorm_stat_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch1, engine->get_gn_stats(), ch, spatial);
|
| 240 |
+
fast_groupnorm_apply_kernel<<<dim3(8, 8), 256, 0, stream>>>(d_scratch1, engine->get_gn_stats(), g_ctx, b_ctx_gn, d_ctx, ch, spatial, true);
|
| 241 |
+
|
| 242 |
+
// 3. agent_proj (3x3 on ctx) -> d_scratch1
|
| 243 |
+
float* w_ap = engine->get_tensor(prefix + "agent_proj.weight");
|
| 244 |
+
float* b_ap = engine->get_tensor(prefix + "agent_proj.bias");
|
| 245 |
+
conv2d_3x3_forward(handle, stream, d_ctx, w_ap, b_ap, d_scratch1, d_im2col, ch, H, W, ch, 1, 1);
|
| 246 |
+
|
| 247 |
+
// 4. agent_gate (1x1 on ctx) -> d_agent_out (temporary gate buffer)
|
| 248 |
+
float* w_ag = engine->get_tensor(prefix + "agent_gate.weight");
|
| 249 |
+
float* b_ag = engine->get_tensor(prefix + "agent_gate.bias");
|
| 250 |
+
conv2d_1x1_forward(handle, stream, d_ctx, w_ag, b_ag, d_agent_out, ch, spatial, ch);
|
| 251 |
+
|
| 252 |
+
// f_agent_out = d_agent + d_scratch1 * sigmoid(d_agent_out)
|
| 253 |
+
gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>(
|
| 254 |
+
d_agent, d_scratch1, d_agent_out, d_agent_out, ch * spatial
|
| 255 |
+
);
|
| 256 |
+
|
| 257 |
+
// 5. wrist_proj (3x3 on ctx) -> d_scratch1
|
| 258 |
+
float* w_wp = engine->get_tensor(prefix + "wrist_proj.weight");
|
| 259 |
+
float* b_wp = engine->get_tensor(prefix + "wrist_proj.bias");
|
| 260 |
+
conv2d_3x3_forward(handle, stream, d_ctx, w_wp, b_wp, d_scratch1, d_im2col, ch, H, W, ch, 1, 1);
|
| 261 |
+
|
| 262 |
+
// 6. wrist_gate (1x1 on ctx) -> d_wrist_out (temporary gate buffer)
|
| 263 |
+
float* w_wg = engine->get_tensor(prefix + "wrist_gate.weight");
|
| 264 |
+
float* b_wg = engine->get_tensor(prefix + "wrist_gate.bias");
|
| 265 |
+
conv2d_1x1_forward(handle, stream, d_ctx, w_wg, b_wg, d_wrist_out, ch, spatial, ch);
|
| 266 |
+
|
| 267 |
+
// f_wrist_out = d_wrist + d_scratch1 * sigmoid(d_wrist_out)
|
| 268 |
+
gated_residual_kernel<<<(ch * spatial + 255) / 256, 256, 0, stream>>>(
|
| 269 |
+
d_wrist, d_scratch1, d_wrist_out, d_wrist_out, ch * spatial
|
| 270 |
+
);
|
| 271 |
+
|
| 272 |
+
// 7. f_mixed = cat(f_agent_out, f_wrist_out)
|
| 273 |
+
cudaMemcpyAsync(d_mixed_out, d_agent_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
|
| 274 |
+
cudaMemcpyAsync(d_mixed_out + ch * spatial, d_wrist_out, ch * spatial * sizeof(float), cudaMemcpyDeviceToDevice, stream);
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
// -------------------------------------------------------------
|
| 278 |
+
// Multi-Head Attention Pooling Forward (Fused Projections & Precomputed Q)
|
| 279 |
+
// -------------------------------------------------------------
|
| 280 |
+
|
| 281 |
+
void attn_pool_forward(
|
| 282 |
+
cublasHandle_t handle,
|
| 283 |
+
cudaStream_t stream,
|
| 284 |
+
const float* d_mixed,
|
| 285 |
+
float* d_out_tokens_4x256,
|
| 286 |
+
float* d_k_proj, // [N, 256]
|
| 287 |
+
float* d_v_proj, // [N, 256]
|
| 288 |
+
float* d_k_head, // [4, N, 64]
|
| 289 |
+
float* d_v_head, // [4, N, 64]
|
| 290 |
+
float* d_scores, // [4, 4, N]
|
| 291 |
+
float* d_ctx_head, // [4, 4, 64]
|
| 292 |
+
float* d_context, // [4, 256]
|
| 293 |
+
int stage,
|
| 294 |
+
ParvulusEngine* engine,
|
| 295 |
+
int in_c, int H, int W
|
| 296 |
+
) {
|
| 297 |
+
int N = H * W;
|
| 298 |
+
float alpha = 1.0f, beta = 0.0f;
|
| 299 |
+
|
| 300 |
+
// 1. Fused K projection directly from d_mixed (eliminates in_proj GEMM & bias)
|
| 301 |
+
float* w_k_fused = engine->get_attn_wk_fused(stage);
|
| 302 |
+
float* b_k_fused = engine->get_attn_bk_fused(stage);
|
| 303 |
+
cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_k_fused, in_c, d_mixed, N, &beta, d_k_proj, 256);
|
| 304 |
+
add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_k_proj, b_k_fused, N, 256);
|
| 305 |
+
transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>(
|
| 306 |
+
d_k_proj, d_k_head, N, 4, 64
|
| 307 |
+
);
|
| 308 |
+
|
| 309 |
+
// 2. Fused V projection directly from d_mixed (eliminates in_proj GEMM & bias)
|
| 310 |
+
float* w_v_fused = engine->get_attn_wv_fused(stage);
|
| 311 |
+
float* b_v_fused = engine->get_attn_bv_fused(stage);
|
| 312 |
+
cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_T, 256, N, in_c, &alpha, w_v_fused, in_c, d_mixed, N, &beta, d_v_proj, 256);
|
| 313 |
+
add_bias_across_tokens_kernel<<<(N * 256 + 255) / 256, 256, 0, stream>>>(d_v_proj, b_v_fused, N, 256);
|
| 314 |
+
transpose_interleaved_to_head_kernel<<<(N * 4 * 64 + 255) / 256, 256, 0, stream>>>(
|
| 315 |
+
d_v_proj, d_v_head, N, 4, 64
|
| 316 |
+
);
|
| 317 |
+
|
| 318 |
+
// 3. Precomputed Q head (Zero FLOPs, Zero runtime GEMM)
|
| 319 |
+
float* d_q_head = engine->get_attn_q_head(stage);
|
| 320 |
+
|
| 321 |
+
// 4. Batched GEMM: Scores = Q * K^T * 0.125
|
| 322 |
+
float scale = 0.125f;
|
| 323 |
+
cublasSgemmStridedBatched(
|
| 324 |
+
handle,
|
| 325 |
+
CUBLAS_OP_T, CUBLAS_OP_N,
|
| 326 |
+
N, 4, 64,
|
| 327 |
+
&scale,
|
| 328 |
+
d_k_head, 64, 64 * N,
|
| 329 |
+
d_q_head, 64, 4 * 64,
|
| 330 |
+
&beta,
|
| 331 |
+
d_scores, N, 4 * N,
|
| 332 |
+
4
|
| 333 |
+
);
|
| 334 |
+
|
| 335 |
+
// 5. Softmax over N keys for each head & query
|
| 336 |
+
attn_pool_softmax_kernel<<<dim3(4, 4), 256, 0, stream>>>(d_scores, N);
|
| 337 |
+
|
| 338 |
+
// 6. Context_h = Scores_h * V_h
|
| 339 |
+
// Scores_h: (N x 4 col-major)
|
| 340 |
+
// V_h: (64 x N col-major, lda = 64)
|
| 341 |
+
// Context_h = V_h * Scores_h: (64 x N) x (N x 4) = (64 x 4 col-major, ldc = 64)
|
| 342 |
+
cublasSgemmStridedBatched(
|
| 343 |
+
handle,
|
| 344 |
+
CUBLAS_OP_N, CUBLAS_OP_N,
|
| 345 |
+
64, 4, N,
|
| 346 |
+
&alpha,
|
| 347 |
+
d_v_head, 64, 64 * N,
|
| 348 |
+
d_scores, N, 4 * N,
|
| 349 |
+
&beta,
|
| 350 |
+
d_ctx_head, 64, 4 * 64,
|
| 351 |
+
4
|
| 352 |
+
);
|
| 353 |
+
|
| 354 |
+
// Transpose context from head-major [4 heads, 4 queries, 64 dims] to interleaved [4 queries, 256 dims]
|
| 355 |
+
transpose_head_to_interleaved_kernel<<<(4 * 4 * 64 + 255) / 256, 256, 0, stream>>>(
|
| 356 |
+
d_ctx_head, d_context, 4, 4, 64
|
| 357 |
+
);
|
| 358 |
+
|
| 359 |
+
// 7. Out projection:
|
| 360 |
+
std::string prefix = "encoder.attn_poolers." + std::to_string(stage) + ".";
|
| 361 |
+
float* w_out = engine->get_tensor(prefix + "cross_attn.out_proj.weight");
|
| 362 |
+
float* b_out = engine->get_tensor(prefix + "cross_attn.out_proj.bias");
|
| 363 |
+
cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, 256, 4, 256, &alpha, w_out, 256, d_context, 256, &beta, d_out_tokens_4x256, 256);
|
| 364 |
+
add_bias_across_tokens_kernel<<<(4 * 256 + 255) / 256, 256, 0, stream>>>(d_out_tokens_4x256, b_out, 4, 256);
|
| 365 |
+
|
| 366 |
+
// 8. LayerNorm(256)
|
| 367 |
+
float* g_norm = engine->get_tensor(prefix + "norm.weight");
|
| 368 |
+
float* b_norm = engine->get_tensor(prefix + "norm.bias");
|
| 369 |
+
layernorm_256_kernel<<<4, 256, 0, stream>>>(d_out_tokens_4x256, g_norm, b_norm, d_out_tokens_4x256, 4);
|
| 370 |
+
}
|
| 371 |
+
|
| 372 |
+
// -----------------------------------------------------------------------------
|
| 373 |
+
// Complete DualView Vision Encoder Forward Pass
|
| 374 |
+
// -----------------------------------------------------------------------------
|
| 375 |
+
|
| 376 |
+
void dual_view_encoder_forward(
|
| 377 |
+
ParvulusEngine* engine,
|
| 378 |
+
cublasHandle_t handle,
|
| 379 |
+
cudaStream_t stream,
|
| 380 |
+
const float* d_obs,
|
| 381 |
+
const float* d_cond,
|
| 382 |
+
float* d_z_vis,
|
| 383 |
+
float* d_scratch_a,
|
| 384 |
+
float* d_scratch_b,
|
| 385 |
+
float* d_scratch_c,
|
| 386 |
+
float* d_scratch_d,
|
| 387 |
+
float* d_im2col,
|
| 388 |
+
float* d_film_ss,
|
| 389 |
+
float* d_pool_k,
|
| 390 |
+
float* d_pool_v,
|
| 391 |
+
float* d_pool_k_head,
|
| 392 |
+
float* d_pool_v_head,
|
| 393 |
+
float* d_pool_q,
|
| 394 |
+
float* d_pool_q_head,
|
| 395 |
+
float* d_pool_scores,
|
| 396 |
+
float* d_pool_ctx_head,
|
| 397 |
+
float* d_pool_context
|
| 398 |
+
) {
|
| 399 |
+
const float* d_agent_raw = d_obs;
|
| 400 |
+
const float* d_wrist_raw = d_obs + 3 * 256 * 256;
|
| 401 |
+
|
| 402 |
+
// 1. Stem convolutions: Conv3x3 (3 -> 48, pad=1, stride=1) on 256x256
|
| 403 |
+
float* w_a_stem = engine->get_tensor("encoder.agent_stem.weight");
|
| 404 |
+
float* b_a_stem = engine->get_tensor("encoder.agent_stem.bias");
|
| 405 |
+
conv2d_3x3_forward(handle, stream, d_agent_raw, w_a_stem, b_a_stem, d_scratch_a, d_im2col, 3, 256, 256, 48, 1, 1);
|
| 406 |
+
|
| 407 |
+
float* w_w_stem = engine->get_tensor("encoder.wrist_stem.weight");
|
| 408 |
+
float* b_w_stem = engine->get_tensor("encoder.wrist_stem.bias");
|
| 409 |
+
conv2d_3x3_forward(handle, stream, d_wrist_raw, w_w_stem, b_w_stem, d_scratch_b, d_im2col, 3, 256, 256, 48, 1, 1);
|
| 410 |
+
|
| 411 |
+
// 2. Downsample stem 0.5x (256x256 -> 128x128):
|
| 412 |
+
avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_a, d_scratch_c, 48, 128, 128);
|
| 413 |
+
avg_pool_2x2_kernel<<<(48 * 128 * 128 + 255) / 256, 256, 0, stream>>>(d_scratch_b, d_scratch_d, 48, 128, 128);
|
| 414 |
+
|
| 415 |
+
int channel_mults[4] = {1, 2, 2, 4};
|
| 416 |
+
int in_channels[4] = {48, 48, 96, 96};
|
| 417 |
+
int resolutions[4] = {128, 64, 32, 16};
|
| 418 |
+
|
| 419 |
+
for (int stage = 0; stage < 4; ++stage) {
|
| 420 |
+
int in_ch = in_channels[stage];
|
| 421 |
+
int out_ch = 48 * channel_mults[stage];
|
| 422 |
+
int H = resolutions[stage];
|
| 423 |
+
int W = resolutions[stage];
|
| 424 |
+
|
| 425 |
+
std::string pfx_a = "encoder.agent_stages." + std::to_string(stage) + ".0.";
|
| 426 |
+
std::string pfx_w = "encoder.wrist_stages." + std::to_string(stage) + ".0.";
|
| 427 |
+
std::string pfx_mix = "encoder.cv_mixers." + std::to_string(stage) + ".";
|
| 428 |
+
std::string pfx_pool = "encoder.attn_poolers." + std::to_string(stage) + ".";
|
| 429 |
+
std::string pfx_down = "encoder.downsamples." + std::to_string(stage) + ".";
|
| 430 |
+
|
| 431 |
+
float* d_stage_tokens = d_z_vis + stage * 4 * 256;
|
| 432 |
+
|
| 433 |
+
if (stage % 2 == 0) {
|
| 434 |
+
// Inputs are in d_scratch_c (agent) and d_scratch_d (wrist)
|
| 435 |
+
// (a) FiLM ResBlocks -> write to d_scratch_a and d_scratch_b
|
| 436 |
+
film_resblock_forward(
|
| 437 |
+
handle, stream, d_scratch_c, d_scratch_a, d_pool_k, d_pool_v,
|
| 438 |
+
d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W
|
| 439 |
+
);
|
| 440 |
+
film_resblock_forward(
|
| 441 |
+
handle, stream, d_scratch_d, d_scratch_b, d_pool_k, d_pool_v,
|
| 442 |
+
d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W
|
| 443 |
+
);
|
| 444 |
+
|
| 445 |
+
// (b) CVMixer: inputs (a, b) -> agent_out (c), wrist_out (d), mixed_out (a)
|
| 446 |
+
cv_mixer_forward(
|
| 447 |
+
handle, stream, d_scratch_a, d_scratch_b, d_scratch_c, d_scratch_d, d_scratch_a,
|
| 448 |
+
d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W
|
| 449 |
+
);
|
| 450 |
+
|
| 451 |
+
// (c) AttnPool2d: mixed is in d_scratch_a
|
| 452 |
+
attn_pool_forward(
|
| 453 |
+
handle, stream, d_scratch_a, d_stage_tokens,
|
| 454 |
+
d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head,
|
| 455 |
+
d_pool_scores, d_pool_ctx_head, d_pool_context,
|
| 456 |
+
stage, engine, 2 * out_ch, H, W
|
| 457 |
+
);
|
| 458 |
+
|
| 459 |
+
// (d) Downsample: read (c, d) -> write to (a, b)
|
| 460 |
+
if (stage < 3) {
|
| 461 |
+
float* w_down = engine->get_tensor(pfx_down + "weight");
|
| 462 |
+
float* b_down = engine->get_tensor(pfx_down + "bias");
|
| 463 |
+
conv2d_3x3_forward(handle, stream, d_scratch_c, w_down, b_down, d_scratch_a, d_im2col, out_ch, H, W, out_ch, 2, 1);
|
| 464 |
+
conv2d_3x3_forward(handle, stream, d_scratch_d, w_down, b_down, d_scratch_b, d_im2col, out_ch, H, W, out_ch, 2, 1);
|
| 465 |
+
}
|
| 466 |
+
} else {
|
| 467 |
+
// Inputs are in d_scratch_a (agent) and d_scratch_b (wrist)
|
| 468 |
+
// (a) FiLM ResBlocks -> write to d_scratch_c and d_scratch_d
|
| 469 |
+
film_resblock_forward(
|
| 470 |
+
handle, stream, d_scratch_a, d_scratch_c, d_pool_k, d_pool_v,
|
| 471 |
+
d_im2col, d_film_ss, d_cond, pfx_a, engine, in_ch, out_ch, H, W
|
| 472 |
+
);
|
| 473 |
+
film_resblock_forward(
|
| 474 |
+
handle, stream, d_scratch_b, d_scratch_d, d_pool_k, d_pool_v,
|
| 475 |
+
d_im2col, d_film_ss, d_cond, pfx_w, engine, in_ch, out_ch, H, W
|
| 476 |
+
);
|
| 477 |
+
|
| 478 |
+
// (b) CVMixer: inputs (c, d) -> agent_out (a), wrist_out (b), mixed_out (c)
|
| 479 |
+
cv_mixer_forward(
|
| 480 |
+
handle, stream, d_scratch_c, d_scratch_d, d_scratch_a, d_scratch_b, d_scratch_c,
|
| 481 |
+
d_pool_k, d_pool_v, d_im2col, pfx_mix, engine, out_ch, H, W
|
| 482 |
+
);
|
| 483 |
+
|
| 484 |
+
// (c) AttnPool2d: mixed is in d_scratch_c
|
| 485 |
+
attn_pool_forward(
|
| 486 |
+
handle, stream, d_scratch_c, d_stage_tokens,
|
| 487 |
+
d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head,
|
| 488 |
+
d_pool_scores, d_pool_ctx_head, d_pool_context,
|
| 489 |
+
stage, engine, 2 * out_ch, H, W
|
| 490 |
+
);
|
| 491 |
+
|
| 492 |
+
// (d) Downsample: read (a, b) -> write to (c, d)
|
| 493 |
+
if (stage < 3) {
|
| 494 |
+
float* w_down = engine->get_tensor(pfx_down + "weight");
|
| 495 |
+
float* b_down = engine->get_tensor(pfx_down + "bias");
|
| 496 |
+
conv2d_3x3_forward(handle, stream, d_scratch_a, w_down, b_down, d_scratch_c, d_im2col, out_ch, H, W, out_ch, 2, 1);
|
| 497 |
+
conv2d_3x3_forward(handle, stream, d_scratch_b, w_down, b_down, d_scratch_d, d_im2col, out_ch, H, W, out_ch, 2, 1);
|
| 498 |
+
}
|
| 499 |
+
}
|
| 500 |
+
}
|
| 501 |
+
}
|
| 502 |
+
|
| 503 |
+
void test_groupnorm(
|
| 504 |
+
const float* d_in, const float* gamma, const float* beta, float* d_out,
|
| 505 |
+
int C, int H, int W, bool silu, cudaStream_t stream
|
| 506 |
+
) {
|
| 507 |
+
groupnorm_2d_kernel<<<8, 256, 0, stream>>>(d_in, gamma, beta, d_out, C, H, W, silu);
|
| 508 |
+
}
|
| 509 |
+
|
| 510 |
+
void test_film_resblock(
|
| 511 |
+
ParvulusEngine* engine,
|
| 512 |
+
cublasHandle_t handle,
|
| 513 |
+
cudaStream_t stream,
|
| 514 |
+
const float* d_in,
|
| 515 |
+
float* d_out,
|
| 516 |
+
float* d_scratch1,
|
| 517 |
+
float* d_scratch2,
|
| 518 |
+
float* d_im2col,
|
| 519 |
+
float* d_film_ss,
|
| 520 |
+
const float* d_cond,
|
| 521 |
+
const std::string& prefix,
|
| 522 |
+
int in_c, int out_c, int H, int W
|
| 523 |
+
) {
|
| 524 |
+
film_resblock_forward(
|
| 525 |
+
handle, stream, d_in, d_out, d_scratch1, d_scratch2, d_im2col, d_film_ss,
|
| 526 |
+
d_cond, prefix, engine, in_c, out_c, H, W
|
| 527 |
+
);
|
| 528 |
+
}
|
| 529 |
+
|
| 530 |
+
void test_attn_pool(
|
| 531 |
+
ParvulusEngine* engine,
|
| 532 |
+
cublasHandle_t handle,
|
| 533 |
+
cudaStream_t stream,
|
| 534 |
+
const float* d_mixed,
|
| 535 |
+
float* d_out_tokens,
|
| 536 |
+
float* d_scratch_b,
|
| 537 |
+
float* d_pool_k,
|
| 538 |
+
float* d_pool_v,
|
| 539 |
+
float* d_pool_k_head,
|
| 540 |
+
float* d_pool_v_head,
|
| 541 |
+
float* d_pool_q,
|
| 542 |
+
float* d_pool_q_head,
|
| 543 |
+
float* d_pool_scores,
|
| 544 |
+
float* d_pool_ctx_head,
|
| 545 |
+
float* d_pool_context,
|
| 546 |
+
const std::string& prefix,
|
| 547 |
+
int in_c, int H, int W
|
| 548 |
+
) {
|
| 549 |
+
attn_pool_forward(
|
| 550 |
+
handle, stream, d_mixed, d_out_tokens,
|
| 551 |
+
d_pool_k, d_pool_v, d_pool_k_head, d_pool_v_head,
|
| 552 |
+
d_pool_scores, d_pool_ctx_head, d_pool_context,
|
| 553 |
+
0, engine, in_c, H, W
|
| 554 |
+
);
|
| 555 |
+
}
|
c_engine/vision_encoder.cuh
ADDED
|
@@ -0,0 +1,560 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <cuda_runtime.h>
|
| 4 |
+
#include <cublas_v2.h>
|
| 5 |
+
#include <cmath>
|
| 6 |
+
|
| 7 |
+
// -----------------------------------------------------------------------------
|
| 8 |
+
// CUDA Kernels for Vision Encoder
|
| 9 |
+
// -----------------------------------------------------------------------------
|
| 10 |
+
|
| 11 |
+
__global__ void im2col_3x3_kernel(
|
| 12 |
+
const float* __restrict__ data_im,
|
| 13 |
+
float* __restrict__ data_col,
|
| 14 |
+
int channels, int height, int width,
|
| 15 |
+
int pad, int stride,
|
| 16 |
+
int out_h, int out_w
|
| 17 |
+
) {
|
| 18 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 19 |
+
int total_spatial = out_h * out_w;
|
| 20 |
+
int total = channels * total_spatial;
|
| 21 |
+
if (idx >= total) return;
|
| 22 |
+
|
| 23 |
+
int spatial_idx = idx % total_spatial;
|
| 24 |
+
int c = idx / total_spatial;
|
| 25 |
+
|
| 26 |
+
int w_out = spatial_idx % out_w;
|
| 27 |
+
int h_out = spatial_idx / out_w;
|
| 28 |
+
|
| 29 |
+
int h_in_base = h_out * stride - pad;
|
| 30 |
+
int w_in_base = w_out * stride - pad;
|
| 31 |
+
|
| 32 |
+
const float* im_c = data_im + c * (height * width);
|
| 33 |
+
int col_channel_base = c * 9;
|
| 34 |
+
|
| 35 |
+
#pragma unroll
|
| 36 |
+
for (int kh = 0; kh < 3; ++kh) {
|
| 37 |
+
int h_in = h_in_base + kh;
|
| 38 |
+
#pragma unroll
|
| 39 |
+
for (int kw = 0; kw < 3; ++kw) {
|
| 40 |
+
int w_in = w_in_base + kw;
|
| 41 |
+
float val = 0.0f;
|
| 42 |
+
if (h_in >= 0 && h_in < height && w_in >= 0 && w_in < width) {
|
| 43 |
+
val = im_c[h_in * width + w_in];
|
| 44 |
+
}
|
| 45 |
+
int col_idx = (col_channel_base + kh * 3 + kw) * total_spatial + spatial_idx;
|
| 46 |
+
data_col[col_idx] = val;
|
| 47 |
+
}
|
| 48 |
+
}
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
__global__ void add_bias_2d_kernel(
|
| 52 |
+
float* __restrict__ data,
|
| 53 |
+
const float* __restrict__ bias,
|
| 54 |
+
int channels, int spatial_dim
|
| 55 |
+
) {
|
| 56 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 57 |
+
int total = channels * spatial_dim;
|
| 58 |
+
if (idx >= total) return;
|
| 59 |
+
int c = idx / spatial_dim;
|
| 60 |
+
data[idx] += bias[c];
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
__global__ void silu_kernel(float* data, int total) {
|
| 64 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 65 |
+
if (idx < total) {
|
| 66 |
+
float x = data[idx];
|
| 67 |
+
data[idx] = x / (1.0f + __expf(-x));
|
| 68 |
+
}
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
__global__ void avg_pool_2x2_kernel(
|
| 72 |
+
const float* __restrict__ in,
|
| 73 |
+
float* __restrict__ out,
|
| 74 |
+
int C, int H_out, int W_out
|
| 75 |
+
) {
|
| 76 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 77 |
+
int total = C * H_out * W_out;
|
| 78 |
+
if (idx >= total) return;
|
| 79 |
+
|
| 80 |
+
int w = idx % W_out;
|
| 81 |
+
int h = (idx / W_out) % H_out;
|
| 82 |
+
int c = idx / (H_out * W_out);
|
| 83 |
+
|
| 84 |
+
int in_w = w * 2;
|
| 85 |
+
int in_h = h * 2;
|
| 86 |
+
int in_w_total = W_out * 2;
|
| 87 |
+
int in_h_total = H_out * 2;
|
| 88 |
+
|
| 89 |
+
const float* in_c = in + c * (in_h_total * in_w_total);
|
| 90 |
+
float val = 0.25f * (
|
| 91 |
+
in_c[in_h * in_w_total + in_w] +
|
| 92 |
+
in_c[in_h * in_w_total + in_w + 1] +
|
| 93 |
+
in_c[(in_h + 1) * in_w_total + in_w] +
|
| 94 |
+
in_c[(in_h + 1) * in_w_total + in_w + 1]
|
| 95 |
+
);
|
| 96 |
+
out[idx] = val;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
__global__ void gated_residual_kernel(
|
| 100 |
+
const float* __restrict__ f_base,
|
| 101 |
+
const float* __restrict__ f_proj,
|
| 102 |
+
const float* __restrict__ f_gate,
|
| 103 |
+
float* __restrict__ f_out,
|
| 104 |
+
int total
|
| 105 |
+
) {
|
| 106 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 107 |
+
if (idx < total) {
|
| 108 |
+
float gate = 1.0f / (1.0f + __expf(-f_gate[idx]));
|
| 109 |
+
f_out[idx] = f_base[idx] + f_proj[idx] * gate;
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
|
| 113 |
+
__global__ void add_residual_kernel(
|
| 114 |
+
const float* __restrict__ a,
|
| 115 |
+
const float* __restrict__ b,
|
| 116 |
+
float* __restrict__ out,
|
| 117 |
+
int total
|
| 118 |
+
) {
|
| 119 |
+
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
| 120 |
+
if (idx < total) {
|
| 121 |
+
out[idx] = a[idx] + b[idx];
|
| 122 |
+
}
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
// GroupNorm(num_groups=8) + optional SiLU
|
| 126 |
+
__global__ void groupnorm_2d_kernel(
|
| 127 |
+
const float* __restrict__ in,
|
| 128 |
+
const float* __restrict__ gamma,
|
| 129 |
+
const float* __restrict__ beta,
|
| 130 |
+
float* __restrict__ out,
|
| 131 |
+
int C, int H, int W,
|
| 132 |
+
bool apply_silu
|
| 133 |
+
) {
|
| 134 |
+
// 1 block per group (8 groups total)
|
| 135 |
+
int group_idx = blockIdx.x;
|
| 136 |
+
int num_groups = 8;
|
| 137 |
+
int channels_per_group = C / num_groups;
|
| 138 |
+
int spatial = H * W;
|
| 139 |
+
int M = channels_per_group * spatial;
|
| 140 |
+
|
| 141 |
+
int c_start = group_idx * channels_per_group;
|
| 142 |
+
int tid = threadIdx.x;
|
| 143 |
+
int bdim = blockDim.x;
|
| 144 |
+
|
| 145 |
+
// Pass 1: compute mean and variance
|
| 146 |
+
float local_sum = 0.0f;
|
| 147 |
+
float local_sq_sum = 0.0f;
|
| 148 |
+
|
| 149 |
+
for (int i = tid; i < M; i += bdim) {
|
| 150 |
+
int c_local = i / spatial;
|
| 151 |
+
int s = i % spatial;
|
| 152 |
+
int c = c_start + c_local;
|
| 153 |
+
float x = in[c * spatial + s];
|
| 154 |
+
local_sum += x;
|
| 155 |
+
local_sq_sum += x * x;
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
// Warp shuffle reduction
|
| 159 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 160 |
+
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
|
| 161 |
+
local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset);
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
__shared__ float s_sum[32];
|
| 165 |
+
__shared__ float s_sq_sum[32];
|
| 166 |
+
int lane = tid % 32;
|
| 167 |
+
int wid = tid / 32;
|
| 168 |
+
|
| 169 |
+
if (lane == 0) {
|
| 170 |
+
s_sum[wid] = local_sum;
|
| 171 |
+
s_sq_sum[wid] = local_sq_sum;
|
| 172 |
+
}
|
| 173 |
+
__syncthreads();
|
| 174 |
+
|
| 175 |
+
if (wid == 0) {
|
| 176 |
+
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
|
| 177 |
+
float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f;
|
| 178 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 179 |
+
val += __shfl_down_sync(0xffffffff, val, offset);
|
| 180 |
+
val_sq += __shfl_down_sync(0xffffffff, val_sq, offset);
|
| 181 |
+
}
|
| 182 |
+
if (lane == 0) {
|
| 183 |
+
s_sum[0] = val;
|
| 184 |
+
s_sq_sum[0] = val_sq;
|
| 185 |
+
}
|
| 186 |
+
}
|
| 187 |
+
__syncthreads();
|
| 188 |
+
|
| 189 |
+
float mean = s_sum[0] / (float)M;
|
| 190 |
+
float var = (s_sq_sum[0] / (float)M) - (mean * mean);
|
| 191 |
+
if (var < 0.0f) var = 0.0f;
|
| 192 |
+
float inv_std = rsqrtf(var + 1e-5f);
|
| 193 |
+
|
| 194 |
+
// Pass 2: Normalize and write output
|
| 195 |
+
for (int i = tid; i < M; i += bdim) {
|
| 196 |
+
int c_local = i / spatial;
|
| 197 |
+
int s = i % spatial;
|
| 198 |
+
int c = c_start + c_local;
|
| 199 |
+
int idx = c * spatial + s;
|
| 200 |
+
|
| 201 |
+
float x = in[idx];
|
| 202 |
+
float norm_x = (x - mean) * inv_std;
|
| 203 |
+
float y = norm_x * gamma[c] + beta[c];
|
| 204 |
+
if (apply_silu) {
|
| 205 |
+
y = y / (1.0f + __expf(-y));
|
| 206 |
+
}
|
| 207 |
+
out[idx] = y;
|
| 208 |
+
}
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
// GroupNorm + FiLM scale/shift + SiLU in a single fused kernel
|
| 212 |
+
__global__ void groupnorm_film_silu_2d_kernel(
|
| 213 |
+
const float* __restrict__ in,
|
| 214 |
+
const float* __restrict__ gamma,
|
| 215 |
+
const float* __restrict__ beta,
|
| 216 |
+
const float* __restrict__ scale,
|
| 217 |
+
const float* __restrict__ shift,
|
| 218 |
+
float* __restrict__ out,
|
| 219 |
+
int C, int H, int W
|
| 220 |
+
) {
|
| 221 |
+
int group_idx = blockIdx.x;
|
| 222 |
+
int num_groups = 8;
|
| 223 |
+
int channels_per_group = C / num_groups;
|
| 224 |
+
int spatial = H * W;
|
| 225 |
+
int M = channels_per_group * spatial;
|
| 226 |
+
|
| 227 |
+
int c_start = group_idx * channels_per_group;
|
| 228 |
+
int tid = threadIdx.x;
|
| 229 |
+
int bdim = blockDim.x;
|
| 230 |
+
|
| 231 |
+
float local_sum = 0.0f;
|
| 232 |
+
float local_sq_sum = 0.0f;
|
| 233 |
+
|
| 234 |
+
for (int i = tid; i < M; i += bdim) {
|
| 235 |
+
int c_local = i / spatial;
|
| 236 |
+
int s = i % spatial;
|
| 237 |
+
int c = c_start + c_local;
|
| 238 |
+
float x = in[c * spatial + s];
|
| 239 |
+
local_sum += x;
|
| 240 |
+
local_sq_sum += x * x;
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 244 |
+
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
|
| 245 |
+
local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset);
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
__shared__ float s_sum[32];
|
| 249 |
+
__shared__ float s_sq_sum[32];
|
| 250 |
+
int lane = tid % 32;
|
| 251 |
+
int wid = tid / 32;
|
| 252 |
+
|
| 253 |
+
if (lane == 0) {
|
| 254 |
+
s_sum[wid] = local_sum;
|
| 255 |
+
s_sq_sum[wid] = local_sq_sum;
|
| 256 |
+
}
|
| 257 |
+
__syncthreads();
|
| 258 |
+
|
| 259 |
+
if (wid == 0) {
|
| 260 |
+
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
|
| 261 |
+
float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f;
|
| 262 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 263 |
+
val += __shfl_down_sync(0xffffffff, val, offset);
|
| 264 |
+
val_sq += __shfl_down_sync(0xffffffff, val_sq, offset);
|
| 265 |
+
}
|
| 266 |
+
if (lane == 0) {
|
| 267 |
+
s_sum[0] = val;
|
| 268 |
+
s_sq_sum[0] = val_sq;
|
| 269 |
+
}
|
| 270 |
+
}
|
| 271 |
+
__syncthreads();
|
| 272 |
+
|
| 273 |
+
float mean = s_sum[0] / (float)M;
|
| 274 |
+
float var = (s_sq_sum[0] / (float)M) - (mean * mean);
|
| 275 |
+
if (var < 0.0f) var = 0.0f;
|
| 276 |
+
float inv_std = rsqrtf(var + 1e-5f);
|
| 277 |
+
|
| 278 |
+
for (int i = tid; i < M; i += bdim) {
|
| 279 |
+
int c_local = i / spatial;
|
| 280 |
+
int s = i % spatial;
|
| 281 |
+
int c = c_start + c_local;
|
| 282 |
+
int idx = c * spatial + s;
|
| 283 |
+
|
| 284 |
+
float x = in[idx];
|
| 285 |
+
float norm_x = (x - mean) * inv_std;
|
| 286 |
+
// GroupNorm affine
|
| 287 |
+
float gn_out = norm_x * gamma[c] + beta[c];
|
| 288 |
+
// FiLM modulation: gn_out * (1 + scale) + shift
|
| 289 |
+
float film_out = gn_out * (1.0f + scale[c]) + shift[c];
|
| 290 |
+
// SiLU
|
| 291 |
+
out[idx] = film_out / (1.0f + __expf(-film_out));
|
| 292 |
+
}
|
| 293 |
+
}
|
| 294 |
+
|
| 295 |
+
// -------------------------------------------------------------
|
| 296 |
+
// Parallel High-Occupancy GroupNorm (64 blocks: 8 groups x 8 workers)
|
| 297 |
+
// -------------------------------------------------------------
|
| 298 |
+
__global__ void fast_groupnorm_stat_kernel(
|
| 299 |
+
const float* __restrict__ in,
|
| 300 |
+
float* __restrict__ group_stats,
|
| 301 |
+
int C, int spatial
|
| 302 |
+
) {
|
| 303 |
+
int group_idx = blockIdx.x;
|
| 304 |
+
int block_in_group = blockIdx.y;
|
| 305 |
+
int num_blocks_per_group = gridDim.y;
|
| 306 |
+
|
| 307 |
+
int channels_per_group = C / 8;
|
| 308 |
+
int M = channels_per_group * spatial;
|
| 309 |
+
int c_start = group_idx * channels_per_group;
|
| 310 |
+
|
| 311 |
+
int tid = threadIdx.x;
|
| 312 |
+
int bdim = blockDim.x;
|
| 313 |
+
|
| 314 |
+
float local_sum = 0.0f;
|
| 315 |
+
float local_sq_sum = 0.0f;
|
| 316 |
+
|
| 317 |
+
int stride = bdim * num_blocks_per_group;
|
| 318 |
+
int start = block_in_group * bdim + tid;
|
| 319 |
+
|
| 320 |
+
for (int i = start; i < M; i += stride) {
|
| 321 |
+
int c_local = i / spatial;
|
| 322 |
+
int s = i % spatial;
|
| 323 |
+
int c = c_start + c_local;
|
| 324 |
+
float x = in[c * spatial + s];
|
| 325 |
+
local_sum += x;
|
| 326 |
+
local_sq_sum += x * x;
|
| 327 |
+
}
|
| 328 |
+
|
| 329 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 330 |
+
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
|
| 331 |
+
local_sq_sum += __shfl_down_sync(0xffffffff, local_sq_sum, offset);
|
| 332 |
+
}
|
| 333 |
+
|
| 334 |
+
__shared__ float s_sum[32];
|
| 335 |
+
__shared__ float s_sq_sum[32];
|
| 336 |
+
int lane = tid % 32;
|
| 337 |
+
int wid = tid / 32;
|
| 338 |
+
|
| 339 |
+
if (lane == 0) {
|
| 340 |
+
s_sum[wid] = local_sum;
|
| 341 |
+
s_sq_sum[wid] = local_sq_sum;
|
| 342 |
+
}
|
| 343 |
+
__syncthreads();
|
| 344 |
+
|
| 345 |
+
if (wid == 0) {
|
| 346 |
+
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
|
| 347 |
+
float val_sq = (tid < (bdim / 32)) ? s_sq_sum[lane] : 0.0f;
|
| 348 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 349 |
+
val += __shfl_down_sync(0xffffffff, val, offset);
|
| 350 |
+
val_sq += __shfl_down_sync(0xffffffff, val_sq, offset);
|
| 351 |
+
}
|
| 352 |
+
if (lane == 0) {
|
| 353 |
+
atomicAdd(&group_stats[group_idx * 2 + 0], val);
|
| 354 |
+
atomicAdd(&group_stats[group_idx * 2 + 1], val_sq);
|
| 355 |
+
}
|
| 356 |
+
}
|
| 357 |
+
}
|
| 358 |
+
|
| 359 |
+
__global__ void fast_groupnorm_apply_kernel(
|
| 360 |
+
const float* __restrict__ in,
|
| 361 |
+
const float* __restrict__ group_stats,
|
| 362 |
+
const float* __restrict__ gamma,
|
| 363 |
+
const float* __restrict__ beta,
|
| 364 |
+
float* __restrict__ out,
|
| 365 |
+
int C, int spatial,
|
| 366 |
+
bool apply_silu
|
| 367 |
+
) {
|
| 368 |
+
int group_idx = blockIdx.x;
|
| 369 |
+
int block_in_group = blockIdx.y;
|
| 370 |
+
int num_blocks_per_group = gridDim.y;
|
| 371 |
+
|
| 372 |
+
int channels_per_group = C / 8;
|
| 373 |
+
int M = channels_per_group * spatial;
|
| 374 |
+
int c_start = group_idx * channels_per_group;
|
| 375 |
+
|
| 376 |
+
float sum = group_stats[group_idx * 2 + 0];
|
| 377 |
+
float sq_sum = group_stats[group_idx * 2 + 1];
|
| 378 |
+
|
| 379 |
+
float mean = sum / (float)M;
|
| 380 |
+
float var = (sq_sum / (float)M) - (mean * mean);
|
| 381 |
+
float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f);
|
| 382 |
+
|
| 383 |
+
int tid = threadIdx.x;
|
| 384 |
+
int bdim = blockDim.x;
|
| 385 |
+
int stride = bdim * num_blocks_per_group;
|
| 386 |
+
int start = block_in_group * bdim + tid;
|
| 387 |
+
|
| 388 |
+
for (int i = start; i < M; i += stride) {
|
| 389 |
+
int c_local = i / spatial;
|
| 390 |
+
int s = i % spatial;
|
| 391 |
+
int c = c_start + c_local;
|
| 392 |
+
int idx = c * spatial + s;
|
| 393 |
+
float x = in[idx];
|
| 394 |
+
float norm = (x - mean) * inv_std;
|
| 395 |
+
float y = norm * gamma[c] + beta[c];
|
| 396 |
+
if (apply_silu) {
|
| 397 |
+
y = y * (1.0f / (1.0f + __expf(-y)));
|
| 398 |
+
}
|
| 399 |
+
out[idx] = y;
|
| 400 |
+
}
|
| 401 |
+
}
|
| 402 |
+
|
| 403 |
+
__global__ void fast_groupnorm_film_silu_apply_kernel(
|
| 404 |
+
const float* __restrict__ in,
|
| 405 |
+
const float* __restrict__ group_stats,
|
| 406 |
+
const float* __restrict__ gamma,
|
| 407 |
+
const float* __restrict__ beta,
|
| 408 |
+
const float* __restrict__ scale,
|
| 409 |
+
const float* __restrict__ shift,
|
| 410 |
+
float* __restrict__ out,
|
| 411 |
+
int C, int spatial
|
| 412 |
+
) {
|
| 413 |
+
int group_idx = blockIdx.x;
|
| 414 |
+
int block_in_group = blockIdx.y;
|
| 415 |
+
int num_blocks_per_group = gridDim.y;
|
| 416 |
+
|
| 417 |
+
int channels_per_group = C / 8;
|
| 418 |
+
int M = channels_per_group * spatial;
|
| 419 |
+
int c_start = group_idx * channels_per_group;
|
| 420 |
+
|
| 421 |
+
float sum = group_stats[group_idx * 2 + 0];
|
| 422 |
+
float sq_sum = group_stats[group_idx * 2 + 1];
|
| 423 |
+
|
| 424 |
+
float mean = sum / (float)M;
|
| 425 |
+
float var = (sq_sum / (float)M) - (mean * mean);
|
| 426 |
+
float inv_std = rsqrtf(fmaxf(var, 0.0f) + 1e-5f);
|
| 427 |
+
|
| 428 |
+
int tid = threadIdx.x;
|
| 429 |
+
int bdim = blockDim.x;
|
| 430 |
+
int stride = bdim * num_blocks_per_group;
|
| 431 |
+
int start = block_in_group * bdim + tid;
|
| 432 |
+
|
| 433 |
+
for (int i = start; i < M; i += stride) {
|
| 434 |
+
int c_local = i / spatial;
|
| 435 |
+
int s = i % spatial;
|
| 436 |
+
int c = c_start + c_local;
|
| 437 |
+
int idx = c * spatial + s;
|
| 438 |
+
|
| 439 |
+
float x = in[idx];
|
| 440 |
+
float norm_x = (x - mean) * inv_std;
|
| 441 |
+
float h = norm_x * gamma[c] + beta[c];
|
| 442 |
+
float modulated = h * (1.0f + scale[c]) + shift[c];
|
| 443 |
+
float out_val = modulated * (1.0f / (1.0f + __expf(-modulated)));
|
| 444 |
+
out[idx] = out_val;
|
| 445 |
+
}
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
// Multi-head attention softmax kernel for AttnPool: 4 heads, 4 queries, N keys
|
| 449 |
+
__global__ void attn_pool_softmax_kernel(
|
| 450 |
+
float* __restrict__ scores,
|
| 451 |
+
int N // number of spatial tokens (e.g. 16384, 4096, 1024, 256)
|
| 452 |
+
) {
|
| 453 |
+
// blockIdx.x: query_idx in [0..3]
|
| 454 |
+
// blockIdx.y: head_idx in [0..3]
|
| 455 |
+
int q_idx = blockIdx.x;
|
| 456 |
+
int h_idx = blockIdx.y;
|
| 457 |
+
int tid = threadIdx.x;
|
| 458 |
+
int bdim = blockDim.x;
|
| 459 |
+
|
| 460 |
+
float* row = scores + (h_idx * 4 + q_idx) * N;
|
| 461 |
+
|
| 462 |
+
// Find max for numerical stability
|
| 463 |
+
float local_max = -1e20f;
|
| 464 |
+
for (int i = tid; i < N; i += bdim) {
|
| 465 |
+
local_max = fmaxf(local_max, row[i]);
|
| 466 |
+
}
|
| 467 |
+
|
| 468 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 469 |
+
local_max = fmaxf(local_max, __shfl_down_sync(0xffffffff, local_max, offset));
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
__shared__ float s_max[32];
|
| 473 |
+
int lane = tid % 32;
|
| 474 |
+
int wid = tid / 32;
|
| 475 |
+
if (lane == 0) s_max[wid] = local_max;
|
| 476 |
+
__syncthreads();
|
| 477 |
+
|
| 478 |
+
if (wid == 0) {
|
| 479 |
+
float val = (tid < (bdim / 32)) ? s_max[lane] : -1e20f;
|
| 480 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 481 |
+
val = fmaxf(val, __shfl_down_sync(0xffffffff, val, offset));
|
| 482 |
+
}
|
| 483 |
+
if (lane == 0) s_max[0] = val;
|
| 484 |
+
}
|
| 485 |
+
__syncthreads();
|
| 486 |
+
float row_max = s_max[0];
|
| 487 |
+
|
| 488 |
+
// Compute exp and sum
|
| 489 |
+
float local_sum = 0.0f;
|
| 490 |
+
for (int i = tid; i < N; i += bdim) {
|
| 491 |
+
float e = __expf(row[i] - row_max);
|
| 492 |
+
row[i] = e;
|
| 493 |
+
local_sum += e;
|
| 494 |
+
}
|
| 495 |
+
|
| 496 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 497 |
+
local_sum += __shfl_down_sync(0xffffffff, local_sum, offset);
|
| 498 |
+
}
|
| 499 |
+
|
| 500 |
+
__shared__ float s_sum[32];
|
| 501 |
+
if (lane == 0) s_sum[wid] = local_sum;
|
| 502 |
+
__syncthreads();
|
| 503 |
+
|
| 504 |
+
if (wid == 0) {
|
| 505 |
+
float val = (tid < (bdim / 32)) ? s_sum[lane] : 0.0f;
|
| 506 |
+
for (int offset = 16; offset > 0; offset /= 2) {
|
| 507 |
+
val += __shfl_down_sync(0xffffffff, val, offset);
|
| 508 |
+
}
|
| 509 |
+
if (lane == 0) s_sum[0] = val;
|
| 510 |
+
}
|
| 511 |
+
__syncthreads();
|
| 512 |
+
float inv_sum = 1.0f / (s_sum[0] + 1e-12f);
|
| 513 |
+
|
| 514 |
+
// Normalize
|
| 515 |
+
for (int i = tid; i < N; i += bdim) {
|
| 516 |
+
row[i] *= inv_sum;
|
| 517 |
+
}
|
| 518 |
+
}
|
| 519 |
+
|
| 520 |
+
// -----------------------------------------------------------------------------
|
| 521 |
+
// Vision Encoder Forward Function Declaration
|
| 522 |
+
// -----------------------------------------------------------------------------
|
| 523 |
+
|
| 524 |
+
class ParvulusEngine;
|
| 525 |
+
|
| 526 |
+
void dual_view_encoder_forward(
|
| 527 |
+
ParvulusEngine* engine,
|
| 528 |
+
cublasHandle_t handle,
|
| 529 |
+
cudaStream_t stream,
|
| 530 |
+
const float* d_obs,
|
| 531 |
+
const float* d_cond,
|
| 532 |
+
float* d_z_vis,
|
| 533 |
+
float* d_scratch_a,
|
| 534 |
+
float* d_scratch_b,
|
| 535 |
+
float* d_scratch_c,
|
| 536 |
+
float* d_scratch_d,
|
| 537 |
+
float* d_im2col,
|
| 538 |
+
float* d_film_ss,
|
| 539 |
+
float* d_pool_k,
|
| 540 |
+
float* d_pool_v,
|
| 541 |
+
float* d_pool_k_head,
|
| 542 |
+
float* d_pool_v_head,
|
| 543 |
+
float* d_pool_q,
|
| 544 |
+
float* d_pool_q_head,
|
| 545 |
+
float* d_pool_scores,
|
| 546 |
+
float* d_pool_ctx_head,
|
| 547 |
+
float* d_pool_context
|
| 548 |
+
);
|
| 549 |
+
|
| 550 |
+
void conv2d_3x3_forward(
|
| 551 |
+
cublasHandle_t handle,
|
| 552 |
+
cudaStream_t stream,
|
| 553 |
+
const float* d_in,
|
| 554 |
+
const float* d_weight,
|
| 555 |
+
const float* d_bias,
|
| 556 |
+
float* d_out,
|
| 557 |
+
float* d_im2col,
|
| 558 |
+
int in_c, int in_h, int in_w,
|
| 559 |
+
int out_c, int stride, int pad
|
| 560 |
+
);
|
c_engine/vision_encoder.h
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <cuda_runtime.h>
|
| 4 |
+
#include <cublas_v2.h>
|
| 5 |
+
#include <string>
|
| 6 |
+
|
| 7 |
+
class ParvulusEngine;
|
| 8 |
+
|
| 9 |
+
void conv2d_3x3_forward(
|
| 10 |
+
cublasHandle_t handle,
|
| 11 |
+
cudaStream_t stream,
|
| 12 |
+
const float* d_in,
|
| 13 |
+
const float* d_weight,
|
| 14 |
+
const float* d_bias,
|
| 15 |
+
float* d_out,
|
| 16 |
+
float* d_im2col,
|
| 17 |
+
int in_c, int in_h, int in_w,
|
| 18 |
+
int out_c, int stride, int pad
|
| 19 |
+
);
|
| 20 |
+
|
| 21 |
+
void conv2d_1x1_forward(
|
| 22 |
+
cublasHandle_t handle,
|
| 23 |
+
cudaStream_t stream,
|
| 24 |
+
const float* d_in,
|
| 25 |
+
const float* d_weight,
|
| 26 |
+
const float* d_bias,
|
| 27 |
+
float* d_out,
|
| 28 |
+
int in_c, int spatial, int out_c
|
| 29 |
+
);
|
| 30 |
+
|
| 31 |
+
void dual_view_encoder_forward(
|
| 32 |
+
ParvulusEngine* engine,
|
| 33 |
+
cublasHandle_t handle,
|
| 34 |
+
cudaStream_t stream,
|
| 35 |
+
const float* d_obs,
|
| 36 |
+
const float* d_cond,
|
| 37 |
+
float* d_z_vis,
|
| 38 |
+
float* d_scratch_a,
|
| 39 |
+
float* d_scratch_b,
|
| 40 |
+
float* d_scratch_c,
|
| 41 |
+
float* d_scratch_d,
|
| 42 |
+
float* d_im2col,
|
| 43 |
+
float* d_film_ss,
|
| 44 |
+
float* d_pool_k,
|
| 45 |
+
float* d_pool_v,
|
| 46 |
+
float* d_pool_k_head,
|
| 47 |
+
float* d_pool_v_head,
|
| 48 |
+
float* d_pool_q,
|
| 49 |
+
float* d_pool_q_head,
|
| 50 |
+
float* d_pool_scores,
|
| 51 |
+
float* d_pool_ctx_head,
|
| 52 |
+
float* d_pool_context
|
| 53 |
+
);
|
config.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model_type": "parvulus",
|
| 3 |
+
"architectures": [
|
| 4 |
+
"ParvulusModel"
|
| 5 |
+
],
|
| 6 |
+
"base_channels": 48,
|
| 7 |
+
"channel_mults": [
|
| 8 |
+
1,
|
| 9 |
+
2,
|
| 10 |
+
2,
|
| 11 |
+
4
|
| 12 |
+
],
|
| 13 |
+
"num_res_blocks": 1,
|
| 14 |
+
"cond_dim": 256,
|
| 15 |
+
"action_dim": 7,
|
| 16 |
+
"chunk_size": 8,
|
| 17 |
+
"tokens_per_layer": 4,
|
| 18 |
+
"resolution": 256,
|
| 19 |
+
"num_ode_steps": 10,
|
| 20 |
+
"text_encoder": {
|
| 21 |
+
"name": "t5",
|
| 22 |
+
"model_name": "google-t5/t5-small",
|
| 23 |
+
"revision": "df1b051c49625cf57a3d0d8d3863ed4d13564fe4",
|
| 24 |
+
"cond_dim": 256,
|
| 25 |
+
"t5_d_model": 512
|
| 26 |
+
},
|
| 27 |
+
"auxiliary_decoders": {
|
| 28 |
+
"video_decoder": {
|
| 29 |
+
"out_channels": 6,
|
| 30 |
+
"num_res_blocks": 1
|
| 31 |
+
},
|
| 32 |
+
"self_decoder": {
|
| 33 |
+
"out_channels": 6,
|
| 34 |
+
"num_res_blocks": 1
|
| 35 |
+
},
|
| 36 |
+
"change_decoder": {
|
| 37 |
+
"out_channels": 2,
|
| 38 |
+
"num_res_blocks": 1
|
| 39 |
+
}
|
| 40 |
+
},
|
| 41 |
+
"action_scale_file": "action_scale.json",
|
| 42 |
+
"torch_dtype": "float32"
|
| 43 |
+
}
|
eval_benchmark_parallel.py
ADDED
|
@@ -0,0 +1,555 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""High-Performance Parallel Benchmark Evaluator for Parvulus across LIBERO.
|
| 3 |
+
|
| 4 |
+
Runs all 40 canonical tasks across 4 suites (50 trials each = 2,000 trials)
|
| 5 |
+
distributed dynamically across multiple GPUs and worker processes.
|
| 6 |
+
Uses atomic lock queue for dynamic load balancing and fault tolerance.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import argparse
|
| 10 |
+
import json
|
| 11 |
+
import os
|
| 12 |
+
import subprocess
|
| 13 |
+
import sys
|
| 14 |
+
import time
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 17 |
+
|
| 18 |
+
import matplotlib.pyplot as plt
|
| 19 |
+
import numpy as np
|
| 20 |
+
|
| 21 |
+
TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"]
|
| 22 |
+
MAX_STEPS = {
|
| 23 |
+
"libero_spatial": 500,
|
| 24 |
+
"libero_object": 500,
|
| 25 |
+
"libero_goal": 500,
|
| 26 |
+
"libero_10": 600,
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
SCRIPT_DIR = Path(__file__).resolve().parent
|
| 30 |
+
PROJECT_ROOT = SCRIPT_DIR
|
| 31 |
+
LIBERO_ROOT = Path("/home/ubuntu/libero-difficulty/LIBERO")
|
| 32 |
+
|
| 33 |
+
# ---------------------------------------------------------------------------
|
| 34 |
+
# Worker process entrypoint
|
| 35 |
+
# ---------------------------------------------------------------------------
|
| 36 |
+
def run_worker(
|
| 37 |
+
worker_id: int,
|
| 38 |
+
gpu_id: int,
|
| 39 |
+
queue_dir: Path,
|
| 40 |
+
results_dir: Path,
|
| 41 |
+
tasks_meta_file: Path,
|
| 42 |
+
num_trials: int,
|
| 43 |
+
chunk_size: int,
|
| 44 |
+
replan_stride: int,
|
| 45 |
+
num_ode_steps: int,
|
| 46 |
+
resolution: int,
|
| 47 |
+
seed_offset: int,
|
| 48 |
+
backend: str = "cuda_c",
|
| 49 |
+
):
|
| 50 |
+
"""Worker process: loops over dynamic task queue and executes rollouts."""
|
| 51 |
+
# Ensure correct environment variables before importing heavy frameworks
|
| 52 |
+
os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id)
|
| 53 |
+
os.environ["MUJOCO_GL"] = "egl"
|
| 54 |
+
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
| 55 |
+
|
| 56 |
+
if str(SCRIPT_DIR) not in sys.path:
|
| 57 |
+
sys.path.insert(0, str(SCRIPT_DIR))
|
| 58 |
+
if LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path:
|
| 59 |
+
sys.path.insert(0, str(LIBERO_ROOT))
|
| 60 |
+
|
| 61 |
+
import torch
|
| 62 |
+
from infer import ParvulusInference
|
| 63 |
+
from libero.libero import benchmark, get_libero_path
|
| 64 |
+
from libero.libero.envs import OffScreenRenderEnv
|
| 65 |
+
|
| 66 |
+
# Load tasks metadata
|
| 67 |
+
with open(tasks_meta_file, "r") as f:
|
| 68 |
+
tasks = json.load(f)
|
| 69 |
+
|
| 70 |
+
device = "cuda:0"
|
| 71 |
+
print(f"[Worker {worker_id} (GPU {gpu_id})] Initializing Parvulus policy on {device} (backend={backend})...", flush=True)
|
| 72 |
+
infer = ParvulusInference.from_pretrained(
|
| 73 |
+
pretrained_path=str(SCRIPT_DIR),
|
| 74 |
+
device=device,
|
| 75 |
+
backend=backend,
|
| 76 |
+
use_kv_cache=True,
|
| 77 |
+
num_ode_steps=num_ode_steps,
|
| 78 |
+
chunk_size=chunk_size,
|
| 79 |
+
replan_stride=replan_stride,
|
| 80 |
+
resolution=resolution,
|
| 81 |
+
)
|
| 82 |
+
|
| 83 |
+
bench_dict = benchmark.get_benchmark_dict()
|
| 84 |
+
loaded_suites: Dict[str, Any] = {}
|
| 85 |
+
|
| 86 |
+
settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
|
| 87 |
+
retreat_action = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0])
|
| 88 |
+
|
| 89 |
+
print(f"[Worker {worker_id} (GPU {gpu_id})] Ready. Polling task queue...", flush=True)
|
| 90 |
+
|
| 91 |
+
while True:
|
| 92 |
+
# Atomic claim of next unclaimed task
|
| 93 |
+
claimed_idx = None
|
| 94 |
+
for idx in range(len(tasks)):
|
| 95 |
+
result_file = results_dir / f"result_task_{idx:02d}.json"
|
| 96 |
+
if result_file.exists():
|
| 97 |
+
continue
|
| 98 |
+
lock_file = queue_dir / f"task_{idx:02d}.lock"
|
| 99 |
+
try:
|
| 100 |
+
fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
| 101 |
+
with os.fdopen(fd, "w") as lf:
|
| 102 |
+
lf.write(f"worker_{worker_id}_pid_{os.getpid()}\n")
|
| 103 |
+
claimed_idx = idx
|
| 104 |
+
break
|
| 105 |
+
except FileExistsError:
|
| 106 |
+
continue
|
| 107 |
+
|
| 108 |
+
if claimed_idx is None:
|
| 109 |
+
# Check if all tasks have results
|
| 110 |
+
all_done = all((results_dir / f"result_task_{i:02d}.json").exists() for i in range(len(tasks)))
|
| 111 |
+
if all_done:
|
| 112 |
+
break
|
| 113 |
+
time.sleep(1.0)
|
| 114 |
+
continue
|
| 115 |
+
|
| 116 |
+
task_info = tasks[claimed_idx]
|
| 117 |
+
suite_name = task_info["suite_name"]
|
| 118 |
+
task_id = task_info["task_id"]
|
| 119 |
+
task_desc = task_info["language"]
|
| 120 |
+
task_name = task_info["task_name"]
|
| 121 |
+
max_steps = MAX_STEPS.get(suite_name, 500)
|
| 122 |
+
|
| 123 |
+
print(
|
| 124 |
+
f"[Worker {worker_id} (GPU {gpu_id})] Claimed Task {claimed_idx:02d} "
|
| 125 |
+
f"[{suite_name} #{task_id}]: '{task_desc[:40]}...'",
|
| 126 |
+
flush=True,
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
if suite_name not in loaded_suites:
|
| 130 |
+
loaded_suites[suite_name] = bench_dict[suite_name]()
|
| 131 |
+
suite = loaded_suites[suite_name]
|
| 132 |
+
task_obj = suite.get_task(task_id)
|
| 133 |
+
|
| 134 |
+
# Load canonical initial states
|
| 135 |
+
with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]):
|
| 136 |
+
init_states = suite.get_task_init_states(task_id)
|
| 137 |
+
|
| 138 |
+
bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file
|
| 139 |
+
env = OffScreenRenderEnv(
|
| 140 |
+
bddl_file_name=str(bddl_file),
|
| 141 |
+
camera_heights=resolution,
|
| 142 |
+
camera_widths=resolution,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
infer.reset(task_description=task_desc)
|
| 146 |
+
|
| 147 |
+
trial_records = []
|
| 148 |
+
task_successes = 0
|
| 149 |
+
task_t0 = time.time()
|
| 150 |
+
|
| 151 |
+
for trial in range(num_trials):
|
| 152 |
+
trial_seed = seed_offset + claimed_idx * 1000 + trial
|
| 153 |
+
env.seed(trial_seed)
|
| 154 |
+
torch.manual_seed(trial_seed)
|
| 155 |
+
np.random.seed(trial_seed)
|
| 156 |
+
|
| 157 |
+
obs = env.reset()
|
| 158 |
+
init_state = init_states[trial % len(init_states)]
|
| 159 |
+
obs = env.set_init_state(init_state)
|
| 160 |
+
|
| 161 |
+
infer.reset(task_description=task_desc)
|
| 162 |
+
|
| 163 |
+
# Settle
|
| 164 |
+
for _ in range(5):
|
| 165 |
+
obs, _, _, _ = env.step(settle_noop)
|
| 166 |
+
|
| 167 |
+
success = False
|
| 168 |
+
step_count = 0
|
| 169 |
+
trial_t0 = time.time()
|
| 170 |
+
|
| 171 |
+
while step_count < max_steps and not success:
|
| 172 |
+
agent_img = obs["agentview_image"]
|
| 173 |
+
wrist_img = obs["robot0_eye_in_hand_image"]
|
| 174 |
+
|
| 175 |
+
action = infer.step(agent_img, wrist_img)
|
| 176 |
+
obs, reward, done, info = env.step(action)
|
| 177 |
+
step_count += 1
|
| 178 |
+
|
| 179 |
+
if done:
|
| 180 |
+
# Probe
|
| 181 |
+
for _ in range(10):
|
| 182 |
+
obs, _, d, _ = env.step(action)
|
| 183 |
+
if d:
|
| 184 |
+
done = True
|
| 185 |
+
# Retreat
|
| 186 |
+
for _ in range(15):
|
| 187 |
+
obs, _, d, _ = env.step(retreat_action)
|
| 188 |
+
# Settle
|
| 189 |
+
for _ in range(5):
|
| 190 |
+
obs, _, d, _ = env.step(np.zeros(7))
|
| 191 |
+
success = bool(done)
|
| 192 |
+
break
|
| 193 |
+
|
| 194 |
+
trial_elapsed = time.time() - trial_t0
|
| 195 |
+
if success:
|
| 196 |
+
task_successes += 1
|
| 197 |
+
|
| 198 |
+
trial_records.append({
|
| 199 |
+
"trial": trial,
|
| 200 |
+
"success": success,
|
| 201 |
+
"steps": step_count,
|
| 202 |
+
"elapsed": round(trial_elapsed, 2),
|
| 203 |
+
})
|
| 204 |
+
|
| 205 |
+
if (trial + 1) % 10 == 0 or trial == num_trials - 1:
|
| 206 |
+
cur_rate = task_successes / (trial + 1) * 100
|
| 207 |
+
print(
|
| 208 |
+
f" [Worker {worker_id} | Task {claimed_idx:02d}] "
|
| 209 |
+
f"Trial {trial+1:02d}/{num_trials}: {task_successes}/{trial+1} ({cur_rate:.1f}%) "
|
| 210 |
+
f"[{trial_elapsed:.2f}s, {step_count} steps]",
|
| 211 |
+
flush=True,
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
env.close()
|
| 215 |
+
task_elapsed = time.time() - task_t0
|
| 216 |
+
|
| 217 |
+
# Save task result atomically
|
| 218 |
+
result_payload = {
|
| 219 |
+
"global_task_idx": claimed_idx,
|
| 220 |
+
"suite_name": suite_name,
|
| 221 |
+
"task_id": task_id,
|
| 222 |
+
"task_name": task_name,
|
| 223 |
+
"task_desc": task_desc,
|
| 224 |
+
"num_trials": num_trials,
|
| 225 |
+
"successes": task_successes,
|
| 226 |
+
"success_rate": task_successes / num_trials,
|
| 227 |
+
"task_elapsed_seconds": round(task_elapsed, 2),
|
| 228 |
+
"worker_id": worker_id,
|
| 229 |
+
"gpu_id": gpu_id,
|
| 230 |
+
"trials": trial_records,
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
tmp_result = results_dir / f"result_task_{claimed_idx:02d}.tmp"
|
| 234 |
+
final_result = results_dir / f"result_task_{claimed_idx:02d}.json"
|
| 235 |
+
with open(tmp_result, "w") as f:
|
| 236 |
+
json.dump(result_payload, f, indent=2)
|
| 237 |
+
os.replace(tmp_result, final_result)
|
| 238 |
+
|
| 239 |
+
print(
|
| 240 |
+
f"[Worker {worker_id} (GPU {gpu_id})] COMPLETED Task {claimed_idx:02d} "
|
| 241 |
+
f"({suite_name} #{task_id}) -> {task_successes}/{num_trials} "
|
| 242 |
+
f"({task_successes/num_trials*100:.1f}%) in {task_elapsed:.1f}s",
|
| 243 |
+
flush=True,
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
+
print(f"[Worker {worker_id} (GPU {gpu_id})] Exiting cleanly.", flush=True)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
# ---------------------------------------------------------------------------
|
| 250 |
+
# Plotting helper
|
| 251 |
+
# ---------------------------------------------------------------------------
|
| 252 |
+
def plot_results(suite_summaries: Dict[str, Dict], overall_rate: float, total_trials: int, save_path: str):
|
| 253 |
+
suites = list(suite_summaries.keys())
|
| 254 |
+
labels = [s.replace("libero_", "").upper() for s in suites]
|
| 255 |
+
scores = [suite_summaries[s]["rate"] * 100 for s in suites]
|
| 256 |
+
|
| 257 |
+
if all(s in suite_summaries for s in ["libero_spatial", "libero_object", "libero_goal"]):
|
| 258 |
+
s3_succ = sum(suite_summaries[s]["successes"] for s in ["libero_spatial", "libero_object", "libero_goal"])
|
| 259 |
+
s3_trials = sum(suite_summaries[s]["trials"] for s in ["libero_spatial", "libero_object", "libero_goal"])
|
| 260 |
+
labels.append("3-SUITE")
|
| 261 |
+
scores.append((s3_succ / s3_trials) * 100)
|
| 262 |
+
|
| 263 |
+
labels.append("OVERALL")
|
| 264 |
+
scores.append(overall_rate * 100)
|
| 265 |
+
|
| 266 |
+
colors = ["#4C72B0", "#55A868", "#C44E52", "#8172B3", "#CCB974", "#64B5CD"][:len(labels)]
|
| 267 |
+
plt.figure(figsize=(max(8, len(labels) * 1.5), 5.5))
|
| 268 |
+
bars = plt.bar(labels, scores, color=colors, width=0.55, edgecolor="black", linewidth=1.2)
|
| 269 |
+
plt.axhline(90, color="red", linestyle="--", linewidth=1.5, label="Gold Reference (90%)")
|
| 270 |
+
plt.ylim(0, 105)
|
| 271 |
+
plt.ylabel("Success Rate (%)", fontsize=12, fontweight="bold")
|
| 272 |
+
plt.title(f"PARVULUS LIBERO Benchmark Evaluation ({total_trials} Trials)", fontsize=14, fontweight="bold")
|
| 273 |
+
plt.grid(axis="y", linestyle=":", alpha=0.6)
|
| 274 |
+
|
| 275 |
+
for bar, score in zip(bars, scores):
|
| 276 |
+
yval = bar.get_height()
|
| 277 |
+
plt.text(bar.get_x() + bar.get_width() / 2.0, yval + 1.2, f"{score:.1f}%", ha="center", va="bottom", fontsize=11, fontweight="bold")
|
| 278 |
+
|
| 279 |
+
plt.legend(loc="upper left")
|
| 280 |
+
plt.tight_layout()
|
| 281 |
+
plt.savefig(save_path, dpi=300)
|
| 282 |
+
plt.close()
|
| 283 |
+
print(f"Benchmark chart saved to {save_path}")
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
# ---------------------------------------------------------------------------
|
| 287 |
+
# Main Orchestrator
|
| 288 |
+
# ---------------------------------------------------------------------------
|
| 289 |
+
def main():
|
| 290 |
+
parser = argparse.ArgumentParser(description="Parvulus 2000-Trial Parallel Benchmark")
|
| 291 |
+
parser.add_argument("--output_dir", type=str, default="./outputs/eval_parvulus_2000")
|
| 292 |
+
parser.add_argument("--num_trials", type=int, default=50, help="Trials per task (default: 50 -> 2,000 total)")
|
| 293 |
+
parser.add_argument("--num_gpus", type=int, default=4, help="Number of GPUs to use")
|
| 294 |
+
parser.add_argument("--workers_per_gpu", type=int, default=3, help="Workers per GPU (default: 3 -> 12 workers)")
|
| 295 |
+
parser.add_argument("--seed", type=int, default=42, help="Base random seed")
|
| 296 |
+
parser.add_argument("--chunk_size", type=int, default=8)
|
| 297 |
+
parser.add_argument("--replan_stride", type=int, default=8)
|
| 298 |
+
parser.add_argument("--num_ode_steps", type=int, default=10)
|
| 299 |
+
parser.add_argument("--resolution", type=int, default=256)
|
| 300 |
+
parser.add_argument("--backend", type=str, default="parvulus", choices=["parvulus", "pure_cuda", "pytorch", "cuda_c", "cuda_graph"])
|
| 301 |
+
# Internal worker flag
|
| 302 |
+
parser.add_argument("--worker_id", type=int, default=None)
|
| 303 |
+
parser.add_argument("--gpu_id", type=int, default=None)
|
| 304 |
+
args = parser.parse_args()
|
| 305 |
+
|
| 306 |
+
# If launched as worker subprocess:
|
| 307 |
+
if args.worker_id is not None:
|
| 308 |
+
out_dir = Path(args.output_dir)
|
| 309 |
+
queue_dir = out_dir / "queue"
|
| 310 |
+
results_dir = out_dir / "results"
|
| 311 |
+
tasks_meta_file = out_dir / "tasks_meta.json"
|
| 312 |
+
run_worker(
|
| 313 |
+
worker_id=args.worker_id,
|
| 314 |
+
gpu_id=args.gpu_id,
|
| 315 |
+
queue_dir=queue_dir,
|
| 316 |
+
results_dir=results_dir,
|
| 317 |
+
tasks_meta_file=tasks_meta_file,
|
| 318 |
+
num_trials=args.num_trials,
|
| 319 |
+
chunk_size=args.chunk_size,
|
| 320 |
+
replan_stride=args.replan_stride,
|
| 321 |
+
num_ode_steps=args.num_ode_steps,
|
| 322 |
+
resolution=args.resolution,
|
| 323 |
+
seed_offset=args.seed,
|
| 324 |
+
backend=args.backend,
|
| 325 |
+
)
|
| 326 |
+
return
|
| 327 |
+
|
| 328 |
+
# Master orchestrator process
|
| 329 |
+
out_dir = Path(args.output_dir)
|
| 330 |
+
queue_dir = out_dir / "queue"
|
| 331 |
+
results_dir = out_dir / "results"
|
| 332 |
+
logs_dir = out_dir / "logs"
|
| 333 |
+
|
| 334 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 335 |
+
queue_dir.mkdir(parents=True, exist_ok=True)
|
| 336 |
+
results_dir.mkdir(parents=True, exist_ok=True)
|
| 337 |
+
logs_dir.mkdir(parents=True, exist_ok=True)
|
| 338 |
+
|
| 339 |
+
# Build canonical tasks list
|
| 340 |
+
if LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path:
|
| 341 |
+
sys.path.insert(0, str(LIBERO_ROOT))
|
| 342 |
+
from libero.libero import benchmark
|
| 343 |
+
bench_dict = benchmark.get_benchmark_dict()
|
| 344 |
+
|
| 345 |
+
tasks_meta = []
|
| 346 |
+
global_idx = 0
|
| 347 |
+
for suite_name in TASK_SUITES:
|
| 348 |
+
suite = bench_dict[suite_name]()
|
| 349 |
+
for task_id in range(suite.n_tasks):
|
| 350 |
+
t = suite.get_task(task_id)
|
| 351 |
+
tasks_meta.append({
|
| 352 |
+
"global_idx": global_idx,
|
| 353 |
+
"suite_name": suite_name,
|
| 354 |
+
"task_id": task_id,
|
| 355 |
+
"task_name": t.name,
|
| 356 |
+
"language": t.language,
|
| 357 |
+
})
|
| 358 |
+
global_idx += 1
|
| 359 |
+
|
| 360 |
+
tasks_meta_file = out_dir / "tasks_meta.json"
|
| 361 |
+
with open(tasks_meta_file, "w") as f:
|
| 362 |
+
json.dump(tasks_meta, f, indent=2)
|
| 363 |
+
|
| 364 |
+
total_tasks = len(tasks_meta)
|
| 365 |
+
total_trials = total_tasks * args.num_trials
|
| 366 |
+
total_workers = args.num_gpus * args.workers_per_gpu
|
| 367 |
+
|
| 368 |
+
print("=" * 75)
|
| 369 |
+
print(" PARVULUS 2,000-TRIAL BENCHMARK ORCHESTRATOR")
|
| 370 |
+
print("=" * 75)
|
| 371 |
+
print(f" Total Tasks: {total_tasks} across {len(TASK_SUITES)} canonical suites")
|
| 372 |
+
print(f" Trials per Task: {args.num_trials}")
|
| 373 |
+
print(f" Total Trials: {total_trials}")
|
| 374 |
+
print(f" Hardware: {args.num_gpus} NVIDIA GPUs x {args.workers_per_gpu} workers = {total_workers} parallel workers")
|
| 375 |
+
print(f" Output Directory: {out_dir}")
|
| 376 |
+
print("=" * 75)
|
| 377 |
+
|
| 378 |
+
# Check for already completed tasks
|
| 379 |
+
completed = [i for i in range(total_tasks) if (results_dir / f"result_task_{i:02d}.json").exists()]
|
| 380 |
+
if completed:
|
| 381 |
+
print(f" [Resume] Found {len(completed)}/{total_tasks} already completed tasks. Skipping them.")
|
| 382 |
+
|
| 383 |
+
# Spawn worker subprocesses
|
| 384 |
+
workers = []
|
| 385 |
+
t_start = time.time()
|
| 386 |
+
|
| 387 |
+
worker_id = 0
|
| 388 |
+
for gpu_id in range(args.num_gpus):
|
| 389 |
+
for _ in range(args.workers_per_gpu):
|
| 390 |
+
log_path = logs_dir / f"worker_{worker_id}_gpu_{gpu_id}.log"
|
| 391 |
+
log_file = open(log_path, "w")
|
| 392 |
+
cmd = [
|
| 393 |
+
sys.executable,
|
| 394 |
+
__file__,
|
| 395 |
+
"--output_dir", str(out_dir),
|
| 396 |
+
"--num_trials", str(args.num_trials),
|
| 397 |
+
"--chunk_size", str(args.chunk_size),
|
| 398 |
+
"--replan_stride", str(args.replan_stride),
|
| 399 |
+
"--num_ode_steps", str(args.num_ode_steps),
|
| 400 |
+
"--resolution", str(args.resolution),
|
| 401 |
+
"--backend", str(args.backend),
|
| 402 |
+
"--seed", str(args.seed),
|
| 403 |
+
"--worker_id", str(worker_id),
|
| 404 |
+
"--gpu_id", str(gpu_id),
|
| 405 |
+
]
|
| 406 |
+
proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT)
|
| 407 |
+
workers.append({
|
| 408 |
+
"worker_id": worker_id,
|
| 409 |
+
"gpu_id": gpu_id,
|
| 410 |
+
"proc": proc,
|
| 411 |
+
"log_file": log_file,
|
| 412 |
+
})
|
| 413 |
+
worker_id += 1
|
| 414 |
+
|
| 415 |
+
print(f"\nSpawned {len(workers)} worker subprocesses across {args.num_gpus} GPUs. Monitoring execution...")
|
| 416 |
+
|
| 417 |
+
# Monitoring loop
|
| 418 |
+
last_print = 0
|
| 419 |
+
while True:
|
| 420 |
+
time.sleep(5.0)
|
| 421 |
+
|
| 422 |
+
# Check workers status
|
| 423 |
+
all_finished = True
|
| 424 |
+
for w in workers:
|
| 425 |
+
ret = w["proc"].poll()
|
| 426 |
+
if ret is None:
|
| 427 |
+
all_finished = False
|
| 428 |
+
|
| 429 |
+
# Scan results
|
| 430 |
+
result_files = list(results_dir.glob("result_task_*.json"))
|
| 431 |
+
num_done = len(result_files)
|
| 432 |
+
|
| 433 |
+
now = time.time()
|
| 434 |
+
if now - last_print >= 15.0 or num_done == total_tasks or all_finished:
|
| 435 |
+
last_print = now
|
| 436 |
+
elapsed = now - t_start
|
| 437 |
+
|
| 438 |
+
# Calculate current stats
|
| 439 |
+
total_succ = 0
|
| 440 |
+
trials_counted = 0
|
| 441 |
+
suite_counts: Dict[str, Dict[str, int]] = {s: {"succ": 0, "trials": 0} for s in TASK_SUITES}
|
| 442 |
+
|
| 443 |
+
for rf in result_files:
|
| 444 |
+
try:
|
| 445 |
+
with open(rf) as f:
|
| 446 |
+
rd = json.load(f)
|
| 447 |
+
s = rd["suite_name"]
|
| 448 |
+
succ = rd["successes"]
|
| 449 |
+
tr = rd["num_trials"]
|
| 450 |
+
total_succ += succ
|
| 451 |
+
trials_counted += tr
|
| 452 |
+
suite_counts[s]["succ"] += succ
|
| 453 |
+
suite_counts[s]["trials"] += tr
|
| 454 |
+
except Exception:
|
| 455 |
+
pass
|
| 456 |
+
|
| 457 |
+
cur_rate = (total_succ / max(1, trials_counted)) * 100
|
| 458 |
+
rate_per_sec = trials_counted / max(1e-5, elapsed)
|
| 459 |
+
remaining_trials = total_trials - trials_counted
|
| 460 |
+
eta_sec = remaining_trials / max(1e-5, rate_per_sec) if rate_per_sec > 0 else 0
|
| 461 |
+
|
| 462 |
+
suite_str = " | ".join([
|
| 463 |
+
f"{s.replace('libero_', '')[:4]}: {suite_counts[s]['succ']}/{suite_counts[s]['trials']}"
|
| 464 |
+
for s in TASK_SUITES
|
| 465 |
+
])
|
| 466 |
+
|
| 467 |
+
print(
|
| 468 |
+
f"[{time.strftime('%H:%M:%S')}] Tasks: {num_done:02d}/{total_tasks} | "
|
| 469 |
+
f"Trials: {trials_counted:4d}/{total_trials} ({cur_rate:5.1f}% SR) | "
|
| 470 |
+
f"Speed: {rate_per_sec:4.1f} tr/s | "
|
| 471 |
+
f"Elapsed: {elapsed/60:4.1f}m | ETA: {eta_sec/60:4.1f}m | [{suite_str}]",
|
| 472 |
+
flush=True,
|
| 473 |
+
)
|
| 474 |
+
|
| 475 |
+
if num_done == total_tasks:
|
| 476 |
+
print("\nAll 40 tasks have completed successfully!")
|
| 477 |
+
break
|
| 478 |
+
|
| 479 |
+
if all_finished and num_done < total_tasks:
|
| 480 |
+
print(f"\n[Warning] All worker processes terminated but only {num_done}/{total_tasks} tasks completed.")
|
| 481 |
+
break
|
| 482 |
+
|
| 483 |
+
# Clean up file handles
|
| 484 |
+
for w in workers:
|
| 485 |
+
w["proc"].wait()
|
| 486 |
+
w["log_file"].close()
|
| 487 |
+
|
| 488 |
+
total_walltime = time.time() - t_start
|
| 489 |
+
print(f"\nExecution finished in {total_walltime/60:.2f} minutes ({total_walltime:.1f} seconds).")
|
| 490 |
+
|
| 491 |
+
# Aggregate full benchmark
|
| 492 |
+
suite_summaries = {s: {"successes": 0, "trials": 0, "rate": 0.0} for s in TASK_SUITES}
|
| 493 |
+
all_results = []
|
| 494 |
+
|
| 495 |
+
for idx in range(total_tasks):
|
| 496 |
+
rf = results_dir / f"result_task_{idx:02d}.json"
|
| 497 |
+
if not rf.exists():
|
| 498 |
+
continue
|
| 499 |
+
with open(rf) as f:
|
| 500 |
+
rd = json.load(f)
|
| 501 |
+
s = rd["suite_name"]
|
| 502 |
+
suite_summaries[s]["successes"] += rd["successes"]
|
| 503 |
+
suite_summaries[s]["trials"] += rd["num_trials"]
|
| 504 |
+
all_results.append(rd)
|
| 505 |
+
|
| 506 |
+
for s in TASK_SUITES:
|
| 507 |
+
trials = suite_summaries[s]["trials"]
|
| 508 |
+
suite_summaries[s]["rate"] = (suite_summaries[s]["successes"] / trials) if trials > 0 else 0.0
|
| 509 |
+
|
| 510 |
+
total_succ = sum(suite_summaries[s]["successes"] for s in TASK_SUITES)
|
| 511 |
+
total_eval_trials = sum(suite_summaries[s]["trials"] for s in TASK_SUITES)
|
| 512 |
+
overall_rate = total_succ / max(1, total_eval_trials)
|
| 513 |
+
|
| 514 |
+
print("\n" + "=" * 75)
|
| 515 |
+
print(f" PARVULUS BENCHMARK RESULTS ({total_eval_trials} Total Trials across 40 Tasks)")
|
| 516 |
+
print("=" * 75)
|
| 517 |
+
for s in TASK_SUITES:
|
| 518 |
+
r = suite_summaries[s]
|
| 519 |
+
print(f" - {s:<18}: {r['successes']:4d} / {r['trials']:4d} ({r['rate'] * 100:6.2f}%)")
|
| 520 |
+
|
| 521 |
+
if all(s in suite_summaries for s in ["libero_spatial", "libero_object", "libero_goal"]):
|
| 522 |
+
s3_succ = sum(suite_summaries[s]["successes"] for s in ["libero_spatial", "libero_object", "libero_goal"])
|
| 523 |
+
s3_trials = sum(suite_summaries[s]["trials"] for s in ["libero_spatial", "libero_object", "libero_goal"])
|
| 524 |
+
s3_rate = s3_succ / max(1, s3_trials)
|
| 525 |
+
print(" " + "-" * 60)
|
| 526 |
+
print(f" - 3-Suite Subtotal : {s3_succ:4d} / {s3_trials:4d} ({s3_rate * 100:6.2f}%)")
|
| 527 |
+
|
| 528 |
+
print(f" - OVERALL BENCHMARK : {total_succ:4d} / {total_eval_trials:4d} ({overall_rate * 100:6.2f}%)")
|
| 529 |
+
print("=" * 75 + "\n")
|
| 530 |
+
|
| 531 |
+
summary_file = out_dir / "benchmark_summary_2000.json"
|
| 532 |
+
chart_file = out_dir / "benchmark_chart_2000.png"
|
| 533 |
+
|
| 534 |
+
with open(summary_file, "w") as f:
|
| 535 |
+
json.dump({
|
| 536 |
+
"overall_success_rate": overall_rate,
|
| 537 |
+
"overall_successes": total_succ,
|
| 538 |
+
"total_trials": total_eval_trials,
|
| 539 |
+
"walltime_minutes": round(total_walltime / 60.0, 2),
|
| 540 |
+
"trials_per_second": round(total_eval_trials / max(1e-5, total_walltime), 2),
|
| 541 |
+
"suite_summaries": suite_summaries,
|
| 542 |
+
"tasks": all_results,
|
| 543 |
+
}, f, indent=2)
|
| 544 |
+
|
| 545 |
+
plot_results(
|
| 546 |
+
suite_summaries=suite_summaries,
|
| 547 |
+
overall_rate=overall_rate,
|
| 548 |
+
total_trials=total_eval_trials,
|
| 549 |
+
save_path=str(chart_file),
|
| 550 |
+
)
|
| 551 |
+
print(f"Summary JSON saved to {summary_file}")
|
| 552 |
+
|
| 553 |
+
|
| 554 |
+
if __name__ == "__main__":
|
| 555 |
+
main()
|
eval_libero.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Single-task and multi-trial LIBERO evaluation runner for Parvulus.
|
| 3 |
+
|
| 4 |
+
Evaluates the Parvulus policy against the canonical LIBERO benchmark suites.
|
| 5 |
+
Supports saving rollout visualization MP4 videos and reporting step-level metrics.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import argparse
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
from typing import Optional, Tuple
|
| 14 |
+
|
| 15 |
+
import imageio
|
| 16 |
+
import numpy as np
|
| 17 |
+
import torch
|
| 18 |
+
|
| 19 |
+
# Ensure repository root is on sys.path
|
| 20 |
+
SCRIPT_DIR = Path(__file__).resolve().parent
|
| 21 |
+
if str(SCRIPT_DIR) not in sys.path:
|
| 22 |
+
sys.path.insert(0, str(SCRIPT_DIR))
|
| 23 |
+
|
| 24 |
+
# Ensure LIBERO is importable if installed adjacent or in standard paths
|
| 25 |
+
LIBERO_ROOT = Path("/home/ubuntu/libero-difficulty/LIBERO")
|
| 26 |
+
if LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path:
|
| 27 |
+
sys.path.insert(0, str(LIBERO_ROOT))
|
| 28 |
+
|
| 29 |
+
from infer import ParvulusInference
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
from libero.libero import benchmark, get_libero_path
|
| 33 |
+
from libero.libero.envs import OffScreenRenderEnv
|
| 34 |
+
except ImportError as e:
|
| 35 |
+
raise ImportError(
|
| 36 |
+
f"Could not import LIBERO ({e}). Ensure LIBERO and its dependencies are installed."
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"]
|
| 40 |
+
MAX_STEPS = {
|
| 41 |
+
"libero_spatial": 500,
|
| 42 |
+
"libero_object": 500,
|
| 43 |
+
"libero_goal": 500,
|
| 44 |
+
"libero_10": 600,
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def run_trial(
|
| 49 |
+
infer: ParvulusInference,
|
| 50 |
+
env: OffScreenRenderEnv,
|
| 51 |
+
task_desc: str,
|
| 52 |
+
init_state: Optional[np.ndarray] = None,
|
| 53 |
+
max_steps: int = 500,
|
| 54 |
+
settle_steps: int = 5,
|
| 55 |
+
save_video_path: Optional[str] = None,
|
| 56 |
+
) -> Tuple[bool, int, float]:
|
| 57 |
+
"""Runs a single simulation rollout trial."""
|
| 58 |
+
infer.reset(task_description=task_desc)
|
| 59 |
+
obs = env.reset()
|
| 60 |
+
if init_state is not None:
|
| 61 |
+
obs = env.set_init_state(init_state)
|
| 62 |
+
|
| 63 |
+
# Initial settling with open gripper
|
| 64 |
+
settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
|
| 65 |
+
for _ in range(settle_steps):
|
| 66 |
+
obs, _, _, _ = env.step(settle_noop)
|
| 67 |
+
|
| 68 |
+
frames = []
|
| 69 |
+
if save_video_path:
|
| 70 |
+
frames.append(obs["agentview_image"][::-1])
|
| 71 |
+
|
| 72 |
+
success = False
|
| 73 |
+
step_count = 0
|
| 74 |
+
t0 = time.time()
|
| 75 |
+
|
| 76 |
+
while step_count < max_steps and not success:
|
| 77 |
+
agent_img = obs["agentview_image"]
|
| 78 |
+
wrist_img = obs["robot0_eye_in_hand_image"]
|
| 79 |
+
|
| 80 |
+
action = infer.step(agent_img, wrist_img)
|
| 81 |
+
obs, reward, done, info = env.step(action)
|
| 82 |
+
step_count += 1
|
| 83 |
+
|
| 84 |
+
if save_video_path:
|
| 85 |
+
frames.append(obs["agentview_image"][::-1])
|
| 86 |
+
|
| 87 |
+
if done:
|
| 88 |
+
# Probe steps to verify stability
|
| 89 |
+
for _ in range(10):
|
| 90 |
+
obs, _, d, _ = env.step(action)
|
| 91 |
+
if save_video_path:
|
| 92 |
+
frames.append(obs["agentview_image"][::-1])
|
| 93 |
+
if d:
|
| 94 |
+
done = True
|
| 95 |
+
|
| 96 |
+
# Retreat steps
|
| 97 |
+
retreat_act = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0])
|
| 98 |
+
for _ in range(15):
|
| 99 |
+
obs, _, d, _ = env.step(retreat_act)
|
| 100 |
+
if save_video_path:
|
| 101 |
+
frames.append(obs["agentview_image"][::-1])
|
| 102 |
+
|
| 103 |
+
# End settle frames
|
| 104 |
+
for _ in range(5):
|
| 105 |
+
obs, _, d, _ = env.step(np.zeros(7))
|
| 106 |
+
if save_video_path:
|
| 107 |
+
frames.append(obs["agentview_image"][::-1])
|
| 108 |
+
|
| 109 |
+
success = bool(done)
|
| 110 |
+
break
|
| 111 |
+
|
| 112 |
+
elapsed = time.time() - t0
|
| 113 |
+
|
| 114 |
+
if save_video_path and frames:
|
| 115 |
+
Path(save_video_path).parent.mkdir(parents=True, exist_ok=True)
|
| 116 |
+
imageio.mimwrite(save_video_path, frames, fps=20)
|
| 117 |
+
print(f"Saved trial video: {save_video_path}")
|
| 118 |
+
|
| 119 |
+
return success, step_count, elapsed
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def parse_args():
|
| 123 |
+
parser = argparse.ArgumentParser(description="Parvulus LIBERO Benchmark Evaluation")
|
| 124 |
+
parser.add_argument("--suite", type=str, default="libero_spatial", choices=TASK_SUITES)
|
| 125 |
+
parser.add_argument("--task_id", type=int, default=0, help="Task index in suite (default: 0)")
|
| 126 |
+
parser.add_argument("--num_trials", type=int, default=1, help="Number of trials to evaluate")
|
| 127 |
+
parser.add_argument("--seed", type=int, default=42, help="Random seed (default: 42)")
|
| 128 |
+
parser.add_argument("--device", type=str, default="cuda", choices=["cuda", "cpu", "xpu", "auto"])
|
| 129 |
+
parser.add_argument("--backend", type=str, default="pytorch", choices=["pytorch", "cuda_c", "cuda_graph", "pure_cuda"], help="Inference backend engine")
|
| 130 |
+
parser.add_argument("--use_kv_cache", action="store_true", default=True, help="Enable fast KV caching")
|
| 131 |
+
parser.add_argument("--num_ode_steps", type=int, default=10, help="Euler integration steps")
|
| 132 |
+
parser.add_argument("--chunk_size", type=int, default=8, help="Action chunk horizon")
|
| 133 |
+
parser.add_argument("--replan_stride", type=int, default=8, help="Receding horizon replan stride")
|
| 134 |
+
parser.add_argument("--resolution", type=int, default=256, help="Camera resolution")
|
| 135 |
+
parser.add_argument("--save_video", action="store_true", help="Save rollout video as MP4")
|
| 136 |
+
parser.add_argument("--output_dir", type=str, default="./eval_results", help="Directory to save videos/results")
|
| 137 |
+
return parser.parse_args()
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def main():
|
| 141 |
+
args = parse_args()
|
| 142 |
+
print("=" * 70)
|
| 143 |
+
print(" PARVULUS Physical AI Model - LIBERO Evaluation")
|
| 144 |
+
print("=" * 70)
|
| 145 |
+
print(f" Suite: {args.suite}")
|
| 146 |
+
print(f" Task ID: {args.task_id}")
|
| 147 |
+
print(f" Trials: {args.num_trials}")
|
| 148 |
+
print(f" Device: {args.device}")
|
| 149 |
+
print(f" KV Cache: {args.use_kv_cache}")
|
| 150 |
+
print(f" ODE Steps: {args.num_ode_steps}")
|
| 151 |
+
print("=" * 70)
|
| 152 |
+
|
| 153 |
+
# Initialize policy
|
| 154 |
+
infer = ParvulusInference.from_pretrained(
|
| 155 |
+
pretrained_path=str(SCRIPT_DIR),
|
| 156 |
+
device=args.device,
|
| 157 |
+
backend=args.backend,
|
| 158 |
+
use_kv_cache=args.use_kv_cache,
|
| 159 |
+
num_ode_steps=args.num_ode_steps,
|
| 160 |
+
chunk_size=args.chunk_size,
|
| 161 |
+
replan_stride=args.replan_stride,
|
| 162 |
+
resolution=args.resolution,
|
| 163 |
+
)
|
| 164 |
+
|
| 165 |
+
# Initialize task suite
|
| 166 |
+
suite_dict = benchmark.get_benchmark_dict()
|
| 167 |
+
if args.suite not in suite_dict:
|
| 168 |
+
raise ValueError(f"Unknown suite {args.suite}. Available: {list(suite_dict.keys())}")
|
| 169 |
+
suite = suite_dict[args.suite]()
|
| 170 |
+
|
| 171 |
+
task = suite.get_task(args.task_id)
|
| 172 |
+
task_desc = task.language
|
| 173 |
+
print(f"\n[Task {args.task_id}]: \"{task_desc}\"")
|
| 174 |
+
|
| 175 |
+
bddl_file = Path(get_libero_path("bddl_files")) / task.problem_folder / task.bddl_file
|
| 176 |
+
env = OffScreenRenderEnv(
|
| 177 |
+
bddl_file_name=str(bddl_file),
|
| 178 |
+
camera_heights=args.resolution,
|
| 179 |
+
camera_widths=args.resolution,
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]):
|
| 183 |
+
init_states = suite.get_task_init_states(args.task_id)
|
| 184 |
+
|
| 185 |
+
max_steps = MAX_STEPS.get(args.suite, 500)
|
| 186 |
+
successes = 0
|
| 187 |
+
|
| 188 |
+
for trial in range(args.num_trials):
|
| 189 |
+
env.seed(args.seed + trial)
|
| 190 |
+
torch.manual_seed(args.seed + trial)
|
| 191 |
+
init_state = init_states[trial % len(init_states)]
|
| 192 |
+
|
| 193 |
+
video_path = None
|
| 194 |
+
if args.save_video:
|
| 195 |
+
video_path = os.path.join(
|
| 196 |
+
args.output_dir, f"{args.suite}_task{args.task_id}_trial{trial}.mp4"
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
print(f"\n--- Running Trial {trial + 1}/{args.num_trials} ---")
|
| 200 |
+
succ, steps, elapsed = run_trial(
|
| 201 |
+
infer=infer,
|
| 202 |
+
env=env,
|
| 203 |
+
task_desc=task_desc,
|
| 204 |
+
init_state=init_state,
|
| 205 |
+
max_steps=max_steps,
|
| 206 |
+
save_video_path=video_path,
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
status_str = "SUCCESS" if succ else "FAILURE"
|
| 210 |
+
print(f" Result: {status_str}")
|
| 211 |
+
print(f" Steps: {steps}/{max_steps}")
|
| 212 |
+
print(f" Latency: {elapsed:.2f}s ({steps / max(1e-5, elapsed):.1f} steps/s)")
|
| 213 |
+
|
| 214 |
+
if succ:
|
| 215 |
+
successes += 1
|
| 216 |
+
|
| 217 |
+
env.close()
|
| 218 |
+
|
| 219 |
+
print("\n" + "=" * 70)
|
| 220 |
+
print(f" SUMMARY: {successes}/{args.num_trials} ({successes / args.num_trials * 100:.1f}%) SUCCESS RATE")
|
| 221 |
+
print("=" * 70)
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
if __name__ == "__main__":
|
| 225 |
+
main()
|
infer.py
ADDED
|
@@ -0,0 +1,297 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Inference client and high-level runtime for Parvulus robot policies."""
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from typing import Dict, List, Optional, Sequence, Tuple, Union
|
| 5 |
+
|
| 6 |
+
import cv2
|
| 7 |
+
import numpy as np
|
| 8 |
+
import torch
|
| 9 |
+
from PIL import Image
|
| 10 |
+
|
| 11 |
+
from action_utils import DEFAULT_SCALE_PATH, denormalize_action, load_action_scale
|
| 12 |
+
from parvulus import ParvulusModel
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class ParvulusInference:
|
| 16 |
+
"""High-level inference runner for Parvulus robot policies."""
|
| 17 |
+
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
model_path: Union[str, Path] = ".",
|
| 21 |
+
scale_path: Union[str, Path] = DEFAULT_SCALE_PATH,
|
| 22 |
+
device: str = "auto",
|
| 23 |
+
use_kv_cache: bool = True,
|
| 24 |
+
load_auxiliary: bool = False,
|
| 25 |
+
chunk_size: int = 8,
|
| 26 |
+
replan_stride: int = 8,
|
| 27 |
+
num_ode_steps: int = 10,
|
| 28 |
+
resolution: int = 256,
|
| 29 |
+
flip_convention: bool = True,
|
| 30 |
+
gamma: Optional[float] = None,
|
| 31 |
+
auto_gamma: bool = False,
|
| 32 |
+
backend: str = "pytorch",
|
| 33 |
+
):
|
| 34 |
+
"""Initializes Parvulus inference runner.
|
| 35 |
+
|
| 36 |
+
Args:
|
| 37 |
+
model_path: Directory containing parvulus.safetensors and config.json.
|
| 38 |
+
scale_path: Path to action_scale.json.
|
| 39 |
+
device: 'cuda', 'cpu', 'xpu', 'hybrid', or 'auto'.
|
| 40 |
+
'hybrid' executes visual perception on Intel iGPU (xpu) and the ODE loop on CPU.
|
| 41 |
+
use_kv_cache: Precomputes visual keys/values for fast 38ms action chunking.
|
| 42 |
+
load_auxiliary: If True, also loads video, change, and self-reconstruction heads.
|
| 43 |
+
chunk_size: Action chunk horizon (default 8).
|
| 44 |
+
replan_stride: Frequency to replan in step() (default 8).
|
| 45 |
+
num_ode_steps: Euler integration steps (default 10).
|
| 46 |
+
resolution: Camera resolution (256x256).
|
| 47 |
+
flip_convention: Applies standard Robosuite/LIBERO camera orientation flip.
|
| 48 |
+
gamma: Explicit exposure correction gamma.
|
| 49 |
+
auto_gamma: Adaptively brightens underexposed manipulation scenes.
|
| 50 |
+
backend: Runtime execution engine: 'pytorch' (standard), 'cuda_c' (pure C ABI),
|
| 51 |
+
or 'cuda_graph' (CUDA Graph replay, 7.68ms / chunk).
|
| 52 |
+
"""
|
| 53 |
+
self.backend = backend
|
| 54 |
+
self.chunk_size = chunk_size
|
| 55 |
+
self.replan_stride = replan_stride
|
| 56 |
+
self.num_ode_steps = num_ode_steps
|
| 57 |
+
self.resolution = resolution
|
| 58 |
+
self.flip_convention = flip_convention
|
| 59 |
+
self.use_kv_cache = use_kv_cache
|
| 60 |
+
self.gamma = gamma
|
| 61 |
+
self.auto_gamma = auto_gamma
|
| 62 |
+
|
| 63 |
+
if backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"):
|
| 64 |
+
from c_engine.infer_c import ParvulusEngine, ParvulusCUDAGraphEngine
|
| 65 |
+
dev_id = 0
|
| 66 |
+
if isinstance(device, str) and "cuda:" in device:
|
| 67 |
+
dev_id = int(device.split(":")[-1])
|
| 68 |
+
elif isinstance(device, torch.device) and device.index is not None:
|
| 69 |
+
dev_id = device.index
|
| 70 |
+
|
| 71 |
+
if backend in ("parvulus", "pure_cuda", "cuda_c"):
|
| 72 |
+
self._c_engine = ParvulusEngine(
|
| 73 |
+
device_id=dev_id,
|
| 74 |
+
chunk_size=chunk_size,
|
| 75 |
+
replan_stride=replan_stride,
|
| 76 |
+
num_ode_steps=num_ode_steps,
|
| 77 |
+
)
|
| 78 |
+
elif backend == "cuda_graph":
|
| 79 |
+
self._c_engine = ParvulusCUDAGraphEngine(
|
| 80 |
+
device_id=dev_id,
|
| 81 |
+
chunk_size=chunk_size,
|
| 82 |
+
replan_stride=replan_stride,
|
| 83 |
+
num_ode_steps=num_ode_steps,
|
| 84 |
+
)
|
| 85 |
+
return
|
| 86 |
+
|
| 87 |
+
self.is_hybrid = False
|
| 88 |
+
if device == "auto":
|
| 89 |
+
if torch.cuda.is_available():
|
| 90 |
+
self.device = torch.device("cuda")
|
| 91 |
+
elif hasattr(torch, "xpu") and torch.xpu.is_available():
|
| 92 |
+
self.device = torch.device("xpu")
|
| 93 |
+
else:
|
| 94 |
+
self.device = torch.device("cpu")
|
| 95 |
+
elif device == "hybrid":
|
| 96 |
+
if not (hasattr(torch, "xpu") and torch.xpu.is_available()):
|
| 97 |
+
print("[Warning] Device 'hybrid' requested but torch.xpu is unavailable. Falling back to CPU.")
|
| 98 |
+
self.device = torch.device("cpu")
|
| 99 |
+
else:
|
| 100 |
+
self.is_hybrid = True
|
| 101 |
+
self.device = torch.device("cpu")
|
| 102 |
+
self.xpu_device = torch.device("xpu")
|
| 103 |
+
else:
|
| 104 |
+
self.device = torch.device(device)
|
| 105 |
+
|
| 106 |
+
eff_gamma = self.gamma if (self.gamma is not None and self.gamma != 1.0) else 0.55
|
| 107 |
+
self._gamma_lut = np.array([((i / 255.0) ** eff_gamma) * 255.0 for i in range(256)]).astype(np.uint8)
|
| 108 |
+
|
| 109 |
+
self.action_min, self.action_max = load_action_scale(scale_path)
|
| 110 |
+
self.action_min = self.action_min.to(self.device)
|
| 111 |
+
self.action_max = self.action_max.to(self.device)
|
| 112 |
+
|
| 113 |
+
self.model = ParvulusModel.from_pretrained(
|
| 114 |
+
pretrained_path=model_path,
|
| 115 |
+
load_auxiliary=load_auxiliary,
|
| 116 |
+
load_text_encoder=True,
|
| 117 |
+
device=self.device,
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
if self.is_hybrid:
|
| 121 |
+
self.model.encoder = self.model.encoder.to(self.xpu_device).eval()
|
| 122 |
+
|
| 123 |
+
self.current_task_desc: Optional[str] = None
|
| 124 |
+
self.current_text_emb: Optional[torch.Tensor] = None
|
| 125 |
+
self.current_text_emb_xpu: Optional[torch.Tensor] = None
|
| 126 |
+
self.action_queue: List[np.ndarray] = []
|
| 127 |
+
self.stride_counter: int = 0
|
| 128 |
+
|
| 129 |
+
@classmethod
|
| 130 |
+
def from_pretrained(cls, pretrained_path: Union[str, Path] = ".", **kwargs) -> "ParvulusInference":
|
| 131 |
+
return cls(model_path=pretrained_path, **kwargs)
|
| 132 |
+
|
| 133 |
+
def preprocess_image(self, img: np.ndarray, target_device: Optional[torch.device] = None) -> torch.Tensor:
|
| 134 |
+
"""Preprocesses [H, W, 3] uint8 image -> [1, 3, res, res] tensor in [-1, 1]."""
|
| 135 |
+
dev = target_device or self.device
|
| 136 |
+
if self.flip_convention:
|
| 137 |
+
img = np.ascontiguousarray(img[::-1, ::-1])
|
| 138 |
+
if img.shape[0] != self.resolution or img.shape[1] != self.resolution:
|
| 139 |
+
img = np.array(Image.fromarray(img).resize((self.resolution, self.resolution), Image.BILINEAR))
|
| 140 |
+
|
| 141 |
+
if self.gamma is not None and self.gamma != 1.0:
|
| 142 |
+
img = cv2.LUT(img, self._gamma_lut)
|
| 143 |
+
elif self.auto_gamma:
|
| 144 |
+
mean_y = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY).mean()
|
| 145 |
+
if mean_y < 112.0:
|
| 146 |
+
img = cv2.LUT(img, self._gamma_lut)
|
| 147 |
+
|
| 148 |
+
t = torch.from_numpy(img).float() / 255.0
|
| 149 |
+
t = t.permute(2, 0, 1) * 2.0 - 1.0
|
| 150 |
+
return t.unsqueeze(0).to(dev)
|
| 151 |
+
|
| 152 |
+
def reset(self, task_description: Optional[str] = None, text_emb: Optional[torch.Tensor] = None):
|
| 153 |
+
"""Resets policy state for a new episode."""
|
| 154 |
+
if self.backend in ("parvulus", "pure_cuda", "cuda_c"):
|
| 155 |
+
if text_emb is not None:
|
| 156 |
+
vec = text_emb.detach().cpu().numpy()
|
| 157 |
+
self._c_engine.reset(task_description=task_description, task_vector=vec)
|
| 158 |
+
elif task_description is not None:
|
| 159 |
+
# If model with text_encoder is loaded, encode on the fly to support perturbations
|
| 160 |
+
if hasattr(self, "model") and hasattr(self.model, "text_encoder") and self.model.text_encoder is not None:
|
| 161 |
+
with torch.no_grad():
|
| 162 |
+
vec = self.model.text_encoder([task_description]).detach().cpu().numpy()[0]
|
| 163 |
+
self._c_engine.reset(task_description=task_description, task_vector=vec)
|
| 164 |
+
else:
|
| 165 |
+
self._c_engine.reset(task_description=task_description)
|
| 166 |
+
else:
|
| 167 |
+
self._c_engine.reset()
|
| 168 |
+
return
|
| 169 |
+
elif self.backend == "cuda_graph":
|
| 170 |
+
self._c_engine.reset(task_description)
|
| 171 |
+
return
|
| 172 |
+
|
| 173 |
+
self.current_task_desc = task_description
|
| 174 |
+
if text_emb is not None:
|
| 175 |
+
self.current_text_emb = text_emb if text_emb.dim() == 2 else text_emb.unsqueeze(0).to(self.device)
|
| 176 |
+
elif task_description is not None:
|
| 177 |
+
with torch.no_grad():
|
| 178 |
+
self.current_text_emb = self.model.text_encoder([task_description]).to(self.device)
|
| 179 |
+
else:
|
| 180 |
+
self.current_text_emb = None
|
| 181 |
+
|
| 182 |
+
if self.is_hybrid and self.current_text_emb is not None:
|
| 183 |
+
self.current_text_emb_xpu = self.current_text_emb.to(self.xpu_device)
|
| 184 |
+
|
| 185 |
+
self.action_queue.clear()
|
| 186 |
+
self.stride_counter = 0
|
| 187 |
+
|
| 188 |
+
@torch.no_grad()
|
| 189 |
+
def predict_chunk(
|
| 190 |
+
self,
|
| 191 |
+
agentview_img: np.ndarray,
|
| 192 |
+
wrist_img: np.ndarray,
|
| 193 |
+
task_description: Optional[str] = None,
|
| 194 |
+
denormalize: bool = True,
|
| 195 |
+
) -> np.ndarray:
|
| 196 |
+
"""Predicts full 8-step continuous action chunk [8, 7]."""
|
| 197 |
+
if self.backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"):
|
| 198 |
+
return self._c_engine.predict_chunk(agentview_img, wrist_img, task_description=task_description)
|
| 199 |
+
|
| 200 |
+
if task_description is not None:
|
| 201 |
+
self.reset(task_description)
|
| 202 |
+
elif self.current_text_emb is None:
|
| 203 |
+
raise ValueError("Call reset(task_description=...) before predicting actions!")
|
| 204 |
+
|
| 205 |
+
if self.is_hybrid:
|
| 206 |
+
agent_t = self.preprocess_image(agentview_img, target_device=self.xpu_device)
|
| 207 |
+
wrist_t = self.preprocess_image(wrist_img, target_device=self.xpu_device)
|
| 208 |
+
obs_xpu = torch.cat([agent_t, wrist_t], dim=1)
|
| 209 |
+
|
| 210 |
+
_, z_vis_xpu = self.model.encoder(obs_xpu, self.current_text_emb_xpu)
|
| 211 |
+
torch.xpu.synchronize()
|
| 212 |
+
|
| 213 |
+
z_vis = z_vis_xpu.to(self.device)
|
| 214 |
+
|
| 215 |
+
z = torch.randn(1, self.chunk_size, self.model.action_dim, device=self.device)
|
| 216 |
+
kv_layers = self.model._get_kv_layers()
|
| 217 |
+
kv_caches = [layer.precompute_kv(z_vis) for layer in kv_layers]
|
| 218 |
+
dt = 1.0 / self.num_ode_steps
|
| 219 |
+
|
| 220 |
+
for step in range(self.num_ode_steps):
|
| 221 |
+
sigma = torch.tensor([step * dt], device=self.device)
|
| 222 |
+
cond = self.current_text_emb + self.model.action_t_mlp(sigma)
|
| 223 |
+
h = self.model.action_decoder.action_in_proj(z) + self.model.action_decoder.pos_emb[:, :self.chunk_size, :]
|
| 224 |
+
for layer, (k_v, v_v) in zip(kv_layers, kv_caches):
|
| 225 |
+
h = layer.forward_with_kv(h, k_v, v_v, cond)
|
| 226 |
+
v_pred = self.model.action_decoder.out_head(self.model.action_decoder.final_norm(h))
|
| 227 |
+
z = z + v_pred * dt
|
| 228 |
+
|
| 229 |
+
chunk_norm = z.clamp(-1.0, 1.0)
|
| 230 |
+
else:
|
| 231 |
+
agent_t = self.preprocess_image(agentview_img)
|
| 232 |
+
wrist_t = self.preprocess_image(wrist_img)
|
| 233 |
+
obs = torch.cat([agent_t, wrist_t], dim=1)
|
| 234 |
+
|
| 235 |
+
chunk_norm = self.model.predict_action_chunk(
|
| 236 |
+
obs,
|
| 237 |
+
text_emb=self.current_text_emb,
|
| 238 |
+
num_ode_steps=self.num_ode_steps,
|
| 239 |
+
use_kv_cache=self.use_kv_cache,
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
if denormalize:
|
| 243 |
+
chunk = denormalize_action(chunk_norm.squeeze(0), self.action_min, self.action_max)
|
| 244 |
+
else:
|
| 245 |
+
chunk = chunk_norm.squeeze(0)
|
| 246 |
+
|
| 247 |
+
return chunk.cpu().numpy()
|
| 248 |
+
|
| 249 |
+
def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray:
|
| 250 |
+
"""Returns single 7-DoF action, replanning every C=replan_stride steps."""
|
| 251 |
+
if self.backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"):
|
| 252 |
+
return self._c_engine.step(agentview_img, wrist_img)
|
| 253 |
+
|
| 254 |
+
if not self.action_queue or self.stride_counter >= self.replan_stride:
|
| 255 |
+
chunk = self.predict_chunk(agentview_img, wrist_img, denormalize=True)
|
| 256 |
+
self.action_queue = list(chunk[: self.replan_stride])
|
| 257 |
+
self.stride_counter = 0
|
| 258 |
+
|
| 259 |
+
action = self.action_queue.pop(0)
|
| 260 |
+
self.stride_counter += 1
|
| 261 |
+
return action
|
| 262 |
+
|
| 263 |
+
@torch.no_grad()
|
| 264 |
+
def rollout_step(
|
| 265 |
+
self,
|
| 266 |
+
agentview_img: np.ndarray,
|
| 267 |
+
wrist_img: np.ndarray,
|
| 268 |
+
task_description: Optional[str] = None,
|
| 269 |
+
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
|
| 270 |
+
"""Auxiliary world-model step: predicts action chunk AND next dual-camera observation.
|
| 271 |
+
|
| 272 |
+
Returns:
|
| 273 |
+
action_chunk: [8, 7] float32 action chunk.
|
| 274 |
+
next_agent_img: [256, 256, 3] uint8 predicted agentview.
|
| 275 |
+
next_wrist_img: [256, 256, 3] uint8 predicted wristview.
|
| 276 |
+
"""
|
| 277 |
+
action_chunk = self.predict_chunk(agentview_img, wrist_img, task_description=task_description)
|
| 278 |
+
|
| 279 |
+
agent_t = self.preprocess_image(agentview_img)
|
| 280 |
+
wrist_t = self.preprocess_image(wrist_img)
|
| 281 |
+
obs = torch.cat([agent_t, wrist_t], dim=1)
|
| 282 |
+
|
| 283 |
+
pred_obs = self.model.predict_next_frame(obs, text_emb=self.current_text_emb, num_ode_steps=self.num_ode_steps)
|
| 284 |
+
pred_obs = ((pred_obs.squeeze(0) + 1.0) / 2.0 * 255.0).clamp(0, 255).byte().permute(1, 2, 0).cpu().numpy()
|
| 285 |
+
|
| 286 |
+
next_agent = pred_obs[:, :, 0:3]
|
| 287 |
+
next_wrist = pred_obs[:, :, 3:6]
|
| 288 |
+
if self.flip_convention:
|
| 289 |
+
next_agent = np.ascontiguousarray(next_agent[::-1, ::-1])
|
| 290 |
+
next_wrist = np.ascontiguousarray(next_wrist[::-1, ::-1])
|
| 291 |
+
|
| 292 |
+
return action_chunk, next_agent, next_wrist
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
ParvulusInfer = ParvulusInference
|
| 296 |
+
|
| 297 |
+
__all__ = ["ParvulusInference", "ParvulusInfer"]
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:afb24ea7ba640e5df4ea38ca52f33940bfe6d80b28f7263d2fc8132c60e9969d
|
| 3 |
+
size 39910428
|
parvulus.py
ADDED
|
@@ -0,0 +1,664 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Parvulus Neural Architecture: Dual-POV Flow Matching Model for Robot Manipulation."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import math
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
|
| 7 |
+
|
| 8 |
+
import torch
|
| 9 |
+
import torch.nn as nn
|
| 10 |
+
import torch.nn.functional as F
|
| 11 |
+
from safetensors.torch import load_file
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def sinusoidal_embedding(t: torch.Tensor, dim: int) -> torch.Tensor:
|
| 15 |
+
half = dim // 2
|
| 16 |
+
freqs = torch.exp(
|
| 17 |
+
-math.log(10000) * torch.arange(half, device=t.device, dtype=torch.float32) / half
|
| 18 |
+
)
|
| 19 |
+
args = t.float()[:, None] * freqs[None, :] * 1000.0
|
| 20 |
+
emb = torch.cat([torch.sin(args), torch.cos(args)], dim=-1)
|
| 21 |
+
if dim % 2:
|
| 22 |
+
emb = F.pad(emb, (0, 1))
|
| 23 |
+
return emb
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class TimestepMLP(nn.Module):
|
| 27 |
+
def __init__(self, cond_dim: int):
|
| 28 |
+
super().__init__()
|
| 29 |
+
self.cond_dim = cond_dim
|
| 30 |
+
self.mlp = nn.Sequential(
|
| 31 |
+
nn.Linear(cond_dim, cond_dim),
|
| 32 |
+
nn.SiLU(),
|
| 33 |
+
nn.Linear(cond_dim, cond_dim),
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
def forward(self, t: torch.Tensor) -> torch.Tensor:
|
| 37 |
+
return self.mlp(sinusoidal_embedding(t, self.cond_dim))
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
class FiLMResBlock2d(nn.Module):
|
| 41 |
+
def __init__(self, in_ch: int, out_ch: int, cond_dim: int, groups: int = 8):
|
| 42 |
+
super().__init__()
|
| 43 |
+
groups_in = min(groups, in_ch)
|
| 44 |
+
groups_out = min(groups, out_ch)
|
| 45 |
+
self.norm1 = nn.GroupNorm(groups_in, in_ch)
|
| 46 |
+
self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1)
|
| 47 |
+
self.norm2 = nn.GroupNorm(groups_out, out_ch)
|
| 48 |
+
self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1)
|
| 49 |
+
self.film = nn.Linear(cond_dim, 2 * out_ch)
|
| 50 |
+
self.skip = nn.Conv2d(in_ch, out_ch, 1) if in_ch != out_ch else nn.Identity()
|
| 51 |
+
|
| 52 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
| 53 |
+
h = self.conv1(F.silu(self.norm1(x)))
|
| 54 |
+
scale, shift = self.film(cond).chunk(2, dim=-1)
|
| 55 |
+
h = self.norm2(h) * (1.0 + scale[:, :, None, None]) + shift[:, :, None, None]
|
| 56 |
+
h = self.conv2(F.silu(h))
|
| 57 |
+
return h + self.skip(x)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class CVMixer(nn.Module):
|
| 61 |
+
"""Bidirectional cross-view mixer with gated contextual residuals."""
|
| 62 |
+
|
| 63 |
+
def __init__(self, ch: int):
|
| 64 |
+
super().__init__()
|
| 65 |
+
self.ch = ch
|
| 66 |
+
self.context_conv = nn.Sequential(
|
| 67 |
+
nn.Conv2d(2 * ch, ch, 1),
|
| 68 |
+
nn.GroupNorm(min(8, ch), ch),
|
| 69 |
+
nn.SiLU(),
|
| 70 |
+
)
|
| 71 |
+
self.agent_proj = nn.Conv2d(ch, ch, 3, padding=1)
|
| 72 |
+
self.agent_gate = nn.Conv2d(ch, ch, 1)
|
| 73 |
+
self.wrist_proj = nn.Conv2d(ch, ch, 3, padding=1)
|
| 74 |
+
self.wrist_gate = nn.Conv2d(ch, ch, 1)
|
| 75 |
+
|
| 76 |
+
def forward(self, f_agent: torch.Tensor, f_wrist: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
| 77 |
+
joint = torch.cat([f_agent, f_wrist], dim=1)
|
| 78 |
+
ctx = self.context_conv(joint)
|
| 79 |
+
|
| 80 |
+
f_agent_out = f_agent + self.agent_proj(ctx) * torch.sigmoid(self.agent_gate(ctx))
|
| 81 |
+
f_wrist_out = f_wrist + self.wrist_proj(ctx) * torch.sigmoid(self.wrist_gate(ctx))
|
| 82 |
+
f_mixed = torch.cat([f_agent_out, f_wrist_out], dim=1)
|
| 83 |
+
return f_agent_out, f_wrist_out, f_mixed
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class AttnPool2d(nn.Module):
|
| 87 |
+
"""Multi-head attention pooling extracting K visual query tokens."""
|
| 88 |
+
|
| 89 |
+
def __init__(self, in_ch: int, embed_dim: int = 256, num_queries: int = 4, num_heads: int = 4):
|
| 90 |
+
super().__init__()
|
| 91 |
+
self.in_proj = nn.Conv2d(in_ch, embed_dim, 1)
|
| 92 |
+
self.queries = nn.Parameter(torch.randn(1, num_queries, embed_dim) * 0.02)
|
| 93 |
+
self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True)
|
| 94 |
+
self.norm = nn.LayerNorm(embed_dim)
|
| 95 |
+
|
| 96 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 97 |
+
b = x.shape[0]
|
| 98 |
+
feats = self.in_proj(x).flatten(2).transpose(1, 2)
|
| 99 |
+
q = self.queries.expand(b, -1, -1)
|
| 100 |
+
out, _ = self.cross_attn(query=q, key=feats, value=feats)
|
| 101 |
+
return self.norm(out)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
class DualViewEncoder(nn.Module):
|
| 105 |
+
"""Hierarchical dual-view vision encoder with cross-view mixing at all stages."""
|
| 106 |
+
|
| 107 |
+
def __init__(
|
| 108 |
+
self,
|
| 109 |
+
base_ch: int = 48,
|
| 110 |
+
channel_mults: Sequence[int] = (1, 2, 2, 4),
|
| 111 |
+
num_res_blocks: int = 1,
|
| 112 |
+
cond_dim: int = 256,
|
| 113 |
+
tokens_per_layer: int = 4,
|
| 114 |
+
):
|
| 115 |
+
super().__init__()
|
| 116 |
+
self.base_ch = base_ch
|
| 117 |
+
self.channel_mults = channel_mults
|
| 118 |
+
self.cond_dim = cond_dim
|
| 119 |
+
|
| 120 |
+
self.agent_stem = nn.Conv2d(3, base_ch, 3, padding=1)
|
| 121 |
+
self.wrist_stem = nn.Conv2d(3, base_ch, 3, padding=1)
|
| 122 |
+
|
| 123 |
+
self.agent_stages = nn.ModuleList()
|
| 124 |
+
self.wrist_stages = nn.ModuleList()
|
| 125 |
+
self.cv_mixers = nn.ModuleList()
|
| 126 |
+
self.downsamples = nn.ModuleList()
|
| 127 |
+
self.attn_poolers = nn.ModuleList()
|
| 128 |
+
self.level_channels: List[int] = []
|
| 129 |
+
|
| 130 |
+
in_ch = base_ch
|
| 131 |
+
for i, mult in enumerate(channel_mults):
|
| 132 |
+
out_ch = base_ch * mult
|
| 133 |
+
a_blocks = nn.ModuleList([
|
| 134 |
+
FiLMResBlock2d(in_ch if j == 0 else out_ch, out_ch, cond_dim)
|
| 135 |
+
for j in range(num_res_blocks)
|
| 136 |
+
])
|
| 137 |
+
self.agent_stages.append(a_blocks)
|
| 138 |
+
|
| 139 |
+
w_blocks = nn.ModuleList([
|
| 140 |
+
FiLMResBlock2d(in_ch if j == 0 else out_ch, out_ch, cond_dim)
|
| 141 |
+
for j in range(num_res_blocks)
|
| 142 |
+
])
|
| 143 |
+
self.wrist_stages.append(w_blocks)
|
| 144 |
+
|
| 145 |
+
self.cv_mixers.append(CVMixer(out_ch))
|
| 146 |
+
self.level_channels.append(2 * out_ch)
|
| 147 |
+
|
| 148 |
+
self.attn_poolers.append(AttnPool2d(2 * out_ch, embed_dim=cond_dim, num_queries=tokens_per_layer))
|
| 149 |
+
|
| 150 |
+
self.downsamples.append(
|
| 151 |
+
nn.Conv2d(out_ch, out_ch, 3, stride=2, padding=1) if i < len(channel_mults) - 1 else nn.Identity()
|
| 152 |
+
)
|
| 153 |
+
in_ch = out_ch
|
| 154 |
+
|
| 155 |
+
def forward(self, x: torch.Tensor, cond: torch.Tensor) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
| 156 |
+
h_agent = self.agent_stem(x[:, 0:3])
|
| 157 |
+
h_wrist = self.wrist_stem(x[:, 3:6])
|
| 158 |
+
|
| 159 |
+
skips = []
|
| 160 |
+
visual_token_list = []
|
| 161 |
+
|
| 162 |
+
for i, (a_stage, w_stage, mixer, pooler, down) in enumerate(
|
| 163 |
+
zip(self.agent_stages, self.wrist_stages, self.cv_mixers, self.attn_poolers, self.downsamples)
|
| 164 |
+
):
|
| 165 |
+
if i == 0:
|
| 166 |
+
h_agent = F.interpolate(h_agent, scale_factor=0.5, mode='bilinear', align_corners=False)
|
| 167 |
+
h_wrist = F.interpolate(h_wrist, scale_factor=0.5, mode='bilinear', align_corners=False)
|
| 168 |
+
|
| 169 |
+
for a_blk, w_blk in zip(a_stage, w_stage):
|
| 170 |
+
h_agent = a_blk(h_agent, cond)
|
| 171 |
+
h_wrist = w_blk(h_wrist, cond)
|
| 172 |
+
|
| 173 |
+
h_agent, h_wrist, f_mixed = mixer(h_agent, h_wrist)
|
| 174 |
+
skips.append(f_mixed)
|
| 175 |
+
|
| 176 |
+
layer_tokens = pooler(f_mixed)
|
| 177 |
+
visual_token_list.append(layer_tokens)
|
| 178 |
+
|
| 179 |
+
h_agent = down(h_agent)
|
| 180 |
+
h_wrist = down(h_wrist)
|
| 181 |
+
|
| 182 |
+
z_vis = torch.cat(visual_token_list, dim=1)
|
| 183 |
+
return skips, z_vis
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
class TransformerDecoderLayer(nn.Module):
|
| 187 |
+
def __init__(self, embed_dim: int = 256, num_heads: int = 8, mlp_ratio: int = 4):
|
| 188 |
+
super().__init__()
|
| 189 |
+
self.norm1 = nn.LayerNorm(embed_dim)
|
| 190 |
+
self.self_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True)
|
| 191 |
+
|
| 192 |
+
self.norm2 = nn.LayerNorm(embed_dim)
|
| 193 |
+
self.cross_attn = nn.MultiheadAttention(embed_dim, num_heads=num_heads, batch_first=True)
|
| 194 |
+
|
| 195 |
+
self.norm3 = nn.LayerNorm(embed_dim)
|
| 196 |
+
self.mlp = nn.Sequential(
|
| 197 |
+
nn.Linear(embed_dim, embed_dim * mlp_ratio),
|
| 198 |
+
nn.GELU(),
|
| 199 |
+
nn.Linear(embed_dim * mlp_ratio, embed_dim),
|
| 200 |
+
)
|
| 201 |
+
self.film = nn.Linear(embed_dim, 2 * embed_dim)
|
| 202 |
+
|
| 203 |
+
def forward(self, x: torch.Tensor, z_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
| 204 |
+
h = x + self.self_attn(query=self.norm1(x), key=self.norm1(x), value=self.norm1(x))[0]
|
| 205 |
+
h = h + self.cross_attn(query=self.norm2(h), key=z_vis, value=z_vis)[0]
|
| 206 |
+
scale, shift = self.film(cond).unsqueeze(1).chunk(2, dim=-1)
|
| 207 |
+
norm_h = self.norm3(h) * (1.0 + scale) + shift
|
| 208 |
+
h = h + self.mlp(norm_h)
|
| 209 |
+
return h
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
class OptimizedKVCacheDecoderLayer(nn.Module):
|
| 213 |
+
def __init__(self, orig_layer: TransformerDecoderLayer):
|
| 214 |
+
super().__init__()
|
| 215 |
+
self.norm1 = orig_layer.norm1
|
| 216 |
+
self.self_attn = orig_layer.self_attn
|
| 217 |
+
self.norm2 = orig_layer.norm2
|
| 218 |
+
self.cross_attn = orig_layer.cross_attn
|
| 219 |
+
self.norm3 = orig_layer.norm3
|
| 220 |
+
self.mlp = orig_layer.mlp
|
| 221 |
+
self.film = orig_layer.film
|
| 222 |
+
|
| 223 |
+
def precompute_kv(self, z_vis: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
| 224 |
+
w = self.cross_attn.in_proj_weight
|
| 225 |
+
b = self.cross_attn.in_proj_bias
|
| 226 |
+
w_k = w[256:512, :]
|
| 227 |
+
b_k = b[256:512] if b is not None else None
|
| 228 |
+
w_v = w[512:768, :]
|
| 229 |
+
b_v = b[512:768] if b is not None else None
|
| 230 |
+
k = F.linear(z_vis, w_k, b_k)
|
| 231 |
+
v = F.linear(z_vis, w_v, b_v)
|
| 232 |
+
return k, v
|
| 233 |
+
|
| 234 |
+
def forward_with_kv(self, x: torch.Tensor, k_vis: torch.Tensor, v_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
| 235 |
+
q1 = self.norm1(x)
|
| 236 |
+
h = x + self.self_attn(q1, q1, q1, need_weights=False)[0]
|
| 237 |
+
|
| 238 |
+
q2 = self.norm2(h)
|
| 239 |
+
w_q = self.cross_attn.in_proj_weight[:256, :]
|
| 240 |
+
b_q = self.cross_attn.in_proj_bias[:256] if self.cross_attn.in_proj_bias is not None else None
|
| 241 |
+
q_proj = F.linear(q2, w_q, b_q)
|
| 242 |
+
|
| 243 |
+
B, S, D = q_proj.shape
|
| 244 |
+
H = 8
|
| 245 |
+
d_k = 32
|
| 246 |
+
q_h = q_proj.view(B, S, H, d_k).transpose(1, 2)
|
| 247 |
+
k_h = k_vis.view(B, 16, H, d_k).transpose(1, 2)
|
| 248 |
+
v_h = v_vis.view(B, 16, H, d_k).transpose(1, 2)
|
| 249 |
+
|
| 250 |
+
attn_out = F.scaled_dot_product_attention(q_h, k_h, v_h)
|
| 251 |
+
attn_out = attn_out.transpose(1, 2).contiguous().view(B, S, D)
|
| 252 |
+
out = F.linear(attn_out, self.cross_attn.out_proj.weight, self.cross_attn.out_proj.bias)
|
| 253 |
+
h = h + out
|
| 254 |
+
|
| 255 |
+
scale, shift = self.film(cond).unsqueeze(1).chunk(2, dim=-1)
|
| 256 |
+
norm_h = self.norm3(h) * (1.0 + scale) + shift
|
| 257 |
+
h = h + self.mlp(norm_h)
|
| 258 |
+
return h
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
class ActionChunkTransformerDecoder(nn.Module):
|
| 262 |
+
def __init__(
|
| 263 |
+
self,
|
| 264 |
+
action_dim: int = 7,
|
| 265 |
+
chunk_size: int = 8,
|
| 266 |
+
embed_dim: int = 256,
|
| 267 |
+
num_layers: int = 4,
|
| 268 |
+
num_heads: int = 8,
|
| 269 |
+
):
|
| 270 |
+
super().__init__()
|
| 271 |
+
self.action_dim = action_dim
|
| 272 |
+
self.chunk_size = chunk_size
|
| 273 |
+
self.embed_dim = embed_dim
|
| 274 |
+
|
| 275 |
+
self.action_in_proj = nn.Linear(action_dim, embed_dim)
|
| 276 |
+
self.pos_emb = nn.Parameter(torch.randn(1, chunk_size, embed_dim) * 0.02)
|
| 277 |
+
self.layers = nn.ModuleList([
|
| 278 |
+
TransformerDecoderLayer(embed_dim, num_heads=num_heads)
|
| 279 |
+
for _ in range(num_layers)
|
| 280 |
+
])
|
| 281 |
+
self.final_norm = nn.LayerNorm(embed_dim)
|
| 282 |
+
self.out_head = nn.Linear(embed_dim, action_dim)
|
| 283 |
+
|
| 284 |
+
def forward(self, z_sigma: torch.Tensor, z_vis: torch.Tensor, cond: torch.Tensor) -> torch.Tensor:
|
| 285 |
+
b, s, d = z_sigma.shape
|
| 286 |
+
h = self.action_in_proj(z_sigma) + self.pos_emb[:, :s, :]
|
| 287 |
+
for layer in self.layers:
|
| 288 |
+
h = layer(h, z_vis, cond)
|
| 289 |
+
return self.out_head(self.final_norm(h))
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
class MultiScaleUNetDecoder(nn.Module):
|
| 293 |
+
def __init__(
|
| 294 |
+
self,
|
| 295 |
+
level_channels: Sequence[int],
|
| 296 |
+
cond_dim: int = 256,
|
| 297 |
+
out_channels: int = 6,
|
| 298 |
+
num_res_blocks: int = 1,
|
| 299 |
+
):
|
| 300 |
+
super().__init__()
|
| 301 |
+
rev_levels = list(reversed(level_channels))
|
| 302 |
+
self.stages = nn.ModuleList()
|
| 303 |
+
self.upsamples = nn.ModuleList()
|
| 304 |
+
|
| 305 |
+
in_ch = rev_levels[0]
|
| 306 |
+
for i, lvl_ch in enumerate(rev_levels):
|
| 307 |
+
blocks = nn.ModuleList()
|
| 308 |
+
for j in range(num_res_blocks):
|
| 309 |
+
b_in = in_ch + (lvl_ch if j == 0 else 0)
|
| 310 |
+
blocks.append(FiLMResBlock2d(b_in, lvl_ch, cond_dim))
|
| 311 |
+
in_ch = lvl_ch
|
| 312 |
+
self.stages.append(blocks)
|
| 313 |
+
self.upsamples.append(
|
| 314 |
+
nn.Sequential(
|
| 315 |
+
nn.Upsample(scale_factor=2, mode='nearest'),
|
| 316 |
+
nn.Conv2d(lvl_ch, lvl_ch, 3, padding=1)
|
| 317 |
+
) if i < len(rev_levels) - 1 else nn.Identity()
|
| 318 |
+
)
|
| 319 |
+
|
| 320 |
+
self.final_upsample = nn.Sequential(
|
| 321 |
+
nn.Upsample(scale_factor=2, mode='nearest'),
|
| 322 |
+
nn.Conv2d(rev_levels[-1], rev_levels[-1], 3, padding=1),
|
| 323 |
+
nn.GroupNorm(min(8, rev_levels[-1]), rev_levels[-1]),
|
| 324 |
+
nn.SiLU(),
|
| 325 |
+
nn.Conv2d(rev_levels[-1], out_channels, 3, padding=1),
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
def forward(self, skips: Sequence[torch.Tensor], cond: torch.Tensor) -> torch.Tensor:
|
| 329 |
+
rev_skips = list(reversed(skips))
|
| 330 |
+
x = rev_skips[0]
|
| 331 |
+
for stage, up, skip in zip(self.stages, self.upsamples, rev_skips):
|
| 332 |
+
for j, block in enumerate(stage):
|
| 333 |
+
if j == 0:
|
| 334 |
+
x = torch.cat([x, skip], dim=1)
|
| 335 |
+
x = block(x, cond)
|
| 336 |
+
x = up(x)
|
| 337 |
+
return self.final_upsample(x)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
class ParvulusOutput(NamedTuple):
|
| 341 |
+
action_flow: Optional[torch.Tensor] = None
|
| 342 |
+
video_flow: Optional[torch.Tensor] = None
|
| 343 |
+
self_recon: Optional[torch.Tensor] = None
|
| 344 |
+
change_pred: Optional[torch.Tensor] = None
|
| 345 |
+
|
| 346 |
+
|
| 347 |
+
class ParvulusModel(nn.Module):
|
| 348 |
+
"""Parvulus: Dual-POV Flow Matching Robot Manipulation Model."""
|
| 349 |
+
|
| 350 |
+
def __init__(
|
| 351 |
+
self,
|
| 352 |
+
base_channels: int = 48,
|
| 353 |
+
channel_mults: Sequence[int] = (1, 2, 2, 4),
|
| 354 |
+
num_res_blocks: int = 1,
|
| 355 |
+
cond_dim: int = 256,
|
| 356 |
+
action_dim: int = 7,
|
| 357 |
+
chunk_size: int = 8,
|
| 358 |
+
text_encoder: Optional[nn.Module] = None,
|
| 359 |
+
use_video_decoder: bool = False,
|
| 360 |
+
use_self_decoder: bool = False,
|
| 361 |
+
use_change_decoder: bool = False,
|
| 362 |
+
):
|
| 363 |
+
super().__init__()
|
| 364 |
+
self.cond_dim = cond_dim
|
| 365 |
+
self.action_dim = action_dim
|
| 366 |
+
self.chunk_size = chunk_size
|
| 367 |
+
self.text_encoder = text_encoder
|
| 368 |
+
|
| 369 |
+
self.encoder = DualViewEncoder(
|
| 370 |
+
base_ch=base_channels,
|
| 371 |
+
channel_mults=channel_mults,
|
| 372 |
+
num_res_blocks=num_res_blocks,
|
| 373 |
+
cond_dim=cond_dim,
|
| 374 |
+
tokens_per_layer=4,
|
| 375 |
+
)
|
| 376 |
+
|
| 377 |
+
self.action_t_mlp = TimestepMLP(cond_dim)
|
| 378 |
+
self.video_t_mlp = TimestepMLP(cond_dim)
|
| 379 |
+
|
| 380 |
+
self.action_decoder = ActionChunkTransformerDecoder(
|
| 381 |
+
action_dim=action_dim,
|
| 382 |
+
chunk_size=chunk_size,
|
| 383 |
+
embed_dim=cond_dim,
|
| 384 |
+
num_layers=4,
|
| 385 |
+
num_heads=8,
|
| 386 |
+
)
|
| 387 |
+
|
| 388 |
+
self.video_decoder = MultiScaleUNetDecoder(
|
| 389 |
+
level_channels=self.encoder.level_channels,
|
| 390 |
+
cond_dim=cond_dim,
|
| 391 |
+
out_channels=6,
|
| 392 |
+
num_res_blocks=num_res_blocks,
|
| 393 |
+
) if use_video_decoder else None
|
| 394 |
+
|
| 395 |
+
self.self_decoder = MultiScaleUNetDecoder(
|
| 396 |
+
level_channels=self.encoder.level_channels,
|
| 397 |
+
cond_dim=cond_dim,
|
| 398 |
+
out_channels=6,
|
| 399 |
+
num_res_blocks=num_res_blocks,
|
| 400 |
+
) if use_self_decoder else None
|
| 401 |
+
|
| 402 |
+
self.change_decoder = MultiScaleUNetDecoder(
|
| 403 |
+
level_channels=self.encoder.level_channels,
|
| 404 |
+
cond_dim=cond_dim,
|
| 405 |
+
out_channels=2,
|
| 406 |
+
num_res_blocks=num_res_blocks,
|
| 407 |
+
) if use_change_decoder else None
|
| 408 |
+
|
| 409 |
+
self._kv_layers: Optional[nn.ModuleList] = None
|
| 410 |
+
|
| 411 |
+
def _get_kv_layers(self) -> nn.ModuleList:
|
| 412 |
+
if self._kv_layers is None:
|
| 413 |
+
self._kv_layers = nn.ModuleList([
|
| 414 |
+
OptimizedKVCacheDecoderLayer(l) for l in self.action_decoder.layers
|
| 415 |
+
])
|
| 416 |
+
return self._kv_layers
|
| 417 |
+
|
| 418 |
+
def _text_cond(
|
| 419 |
+
self,
|
| 420 |
+
texts: Optional[Sequence[str]],
|
| 421 |
+
text_emb: Optional[torch.Tensor],
|
| 422 |
+
batch: int,
|
| 423 |
+
device: torch.device,
|
| 424 |
+
) -> torch.Tensor:
|
| 425 |
+
if text_emb is not None:
|
| 426 |
+
return text_emb
|
| 427 |
+
if texts is not None and self.text_encoder is not None:
|
| 428 |
+
return self.text_encoder(texts)
|
| 429 |
+
return torch.zeros(batch, self.cond_dim, device=device)
|
| 430 |
+
|
| 431 |
+
def forward(
|
| 432 |
+
self,
|
| 433 |
+
obs_input: torch.Tensor,
|
| 434 |
+
z_sigma: Optional[torch.Tensor] = None,
|
| 435 |
+
action_sigma: Optional[torch.Tensor] = None,
|
| 436 |
+
next_target: Optional[torch.Tensor] = None,
|
| 437 |
+
video_tau: Optional[torch.Tensor] = None,
|
| 438 |
+
texts: Optional[Sequence[str]] = None,
|
| 439 |
+
text_emb: Optional[torch.Tensor] = None,
|
| 440 |
+
compute_action: bool = True,
|
| 441 |
+
compute_video: bool = True,
|
| 442 |
+
compute_self_recon: bool = True,
|
| 443 |
+
compute_change: bool = True,
|
| 444 |
+
) -> ParvulusOutput:
|
| 445 |
+
b = obs_input.shape[0]
|
| 446 |
+
device = obs_input.device
|
| 447 |
+
t_emb = self._text_cond(texts, text_emb, b, device)
|
| 448 |
+
|
| 449 |
+
skips_obs, z_vis = self.encoder(obs_input, t_emb)
|
| 450 |
+
|
| 451 |
+
action_flow = None
|
| 452 |
+
if compute_action and z_sigma is not None and action_sigma is not None:
|
| 453 |
+
act_cond = t_emb + self.action_t_mlp(action_sigma)
|
| 454 |
+
action_flow = self.action_decoder(z_sigma, z_vis, act_cond)
|
| 455 |
+
|
| 456 |
+
video_flow = None
|
| 457 |
+
if compute_video and self.video_decoder is not None and video_tau is not None:
|
| 458 |
+
vid_cond = t_emb + self.video_t_mlp(video_tau)
|
| 459 |
+
video_flow = self.video_decoder(skips_obs, vid_cond)
|
| 460 |
+
|
| 461 |
+
self_recon = None
|
| 462 |
+
if compute_self_recon and self.self_decoder is not None:
|
| 463 |
+
self_recon = self.self_decoder(skips_obs, t_emb)
|
| 464 |
+
|
| 465 |
+
change_pred = None
|
| 466 |
+
if compute_change and self.change_decoder is not None:
|
| 467 |
+
change_pred = self.change_decoder(skips_obs, t_emb)
|
| 468 |
+
|
| 469 |
+
return ParvulusOutput(
|
| 470 |
+
action_flow=action_flow,
|
| 471 |
+
video_flow=video_flow,
|
| 472 |
+
self_recon=self_recon,
|
| 473 |
+
change_pred=change_pred,
|
| 474 |
+
)
|
| 475 |
+
|
| 476 |
+
@torch.no_grad()
|
| 477 |
+
def predict_action_chunk(
|
| 478 |
+
self,
|
| 479 |
+
obs: torch.Tensor,
|
| 480 |
+
text_emb: Optional[torch.Tensor] = None,
|
| 481 |
+
texts: Optional[Sequence[str]] = None,
|
| 482 |
+
num_ode_steps: int = 10,
|
| 483 |
+
ode_solver: str = "euler",
|
| 484 |
+
use_kv_cache: bool = False,
|
| 485 |
+
z_init: Optional[torch.Tensor] = None,
|
| 486 |
+
) -> torch.Tensor:
|
| 487 |
+
b = obs.shape[0]
|
| 488 |
+
device = obs.device
|
| 489 |
+
t_emb = self._text_cond(texts, text_emb, b, device)
|
| 490 |
+
|
| 491 |
+
_, z_vis = self.encoder(obs, t_emb)
|
| 492 |
+
|
| 493 |
+
if z_init is not None:
|
| 494 |
+
z = z_init.clone()
|
| 495 |
+
else:
|
| 496 |
+
z = torch.randn(b, self.chunk_size, self.action_dim, device=device)
|
| 497 |
+
|
| 498 |
+
if use_kv_cache and ode_solver == "euler":
|
| 499 |
+
kv_layers = self._get_kv_layers()
|
| 500 |
+
kv_caches = [layer.precompute_kv(z_vis) for layer in kv_layers]
|
| 501 |
+
dt = 1.0 / num_ode_steps
|
| 502 |
+
|
| 503 |
+
for step in range(num_ode_steps):
|
| 504 |
+
sigma = torch.full((b,), step * dt, device=device)
|
| 505 |
+
cond = t_emb + self.action_t_mlp(sigma)
|
| 506 |
+
h = self.action_decoder.action_in_proj(z) + self.action_decoder.pos_emb[:, :self.chunk_size, :]
|
| 507 |
+
for layer, (k_v, v_v) in zip(kv_layers, kv_caches):
|
| 508 |
+
h = layer.forward_with_kv(h, k_v, v_v, cond)
|
| 509 |
+
v_pred = self.action_decoder.out_head(self.action_decoder.final_norm(h))
|
| 510 |
+
z = z + v_pred * dt
|
| 511 |
+
|
| 512 |
+
return z.clamp(-1.0, 1.0)
|
| 513 |
+
|
| 514 |
+
if ode_solver == "euler":
|
| 515 |
+
dt = 1.0 / num_ode_steps
|
| 516 |
+
for step in range(num_ode_steps):
|
| 517 |
+
sigma = torch.full((b,), step * dt, device=device)
|
| 518 |
+
act_cond = t_emb + self.action_t_mlp(sigma)
|
| 519 |
+
v_pred = self.action_decoder(z, z_vis, act_cond)
|
| 520 |
+
z = z + v_pred * dt
|
| 521 |
+
elif ode_solver == "heun":
|
| 522 |
+
dt = 1.0 / num_ode_steps
|
| 523 |
+
for step in range(num_ode_steps):
|
| 524 |
+
t_curr = step * dt
|
| 525 |
+
t_next = (step + 1) * dt
|
| 526 |
+
sigma_curr = torch.full((b,), t_curr, device=device)
|
| 527 |
+
sigma_next = torch.full((b,), t_next, device=device)
|
| 528 |
+
|
| 529 |
+
act_cond_curr = t_emb + self.action_t_mlp(sigma_curr)
|
| 530 |
+
v1 = self.action_decoder(z, z_vis, act_cond_curr)
|
| 531 |
+
z_euler = z + v1 * dt
|
| 532 |
+
|
| 533 |
+
act_cond_next = t_emb + self.action_t_mlp(sigma_next)
|
| 534 |
+
v2 = self.action_decoder(z_euler, z_vis, act_cond_next)
|
| 535 |
+
z = z + 0.5 * dt * (v1 + v2)
|
| 536 |
+
else:
|
| 537 |
+
raise ValueError(f"Unsupported ode_solver: {ode_solver}")
|
| 538 |
+
|
| 539 |
+
return z.clamp(-1.0, 1.0)
|
| 540 |
+
|
| 541 |
+
@torch.no_grad()
|
| 542 |
+
def predict_next_frame(
|
| 543 |
+
self,
|
| 544 |
+
obs: torch.Tensor,
|
| 545 |
+
text_emb: Optional[torch.Tensor] = None,
|
| 546 |
+
texts: Optional[Sequence[str]] = None,
|
| 547 |
+
num_ode_steps: int = 10,
|
| 548 |
+
x_init: Optional[torch.Tensor] = None,
|
| 549 |
+
) -> torch.Tensor:
|
| 550 |
+
if self.video_decoder is None:
|
| 551 |
+
raise RuntimeError("video_decoder is not initialized. Pass use_video_decoder=True when loading model.")
|
| 552 |
+
|
| 553 |
+
b = obs.shape[0]
|
| 554 |
+
device = obs.device
|
| 555 |
+
t_emb = self._text_cond(texts, text_emb, b, device)
|
| 556 |
+
skips, _ = self.encoder(obs, t_emb)
|
| 557 |
+
|
| 558 |
+
if x_init is not None:
|
| 559 |
+
x = x_init.clone()
|
| 560 |
+
else:
|
| 561 |
+
x = torch.randn(b, 6, 256, 256, device=device)
|
| 562 |
+
|
| 563 |
+
dt = 1.0 / num_ode_steps
|
| 564 |
+
for step in range(num_ode_steps):
|
| 565 |
+
tau = torch.full((b,), step * dt, device=device)
|
| 566 |
+
cond = t_emb + self.video_t_mlp(tau)
|
| 567 |
+
v_pred = self.video_decoder(skips, cond)
|
| 568 |
+
x = x + v_pred * dt
|
| 569 |
+
|
| 570 |
+
return x.clamp(-1.0, 1.0)
|
| 571 |
+
|
| 572 |
+
@torch.no_grad()
|
| 573 |
+
def predict_change(
|
| 574 |
+
self,
|
| 575 |
+
obs: torch.Tensor,
|
| 576 |
+
text_emb: Optional[torch.Tensor] = None,
|
| 577 |
+
texts: Optional[Sequence[str]] = None,
|
| 578 |
+
) -> torch.Tensor:
|
| 579 |
+
if self.change_decoder is None:
|
| 580 |
+
raise RuntimeError("change_decoder is not initialized. Pass use_change_decoder=True when loading model.")
|
| 581 |
+
b = obs.shape[0]
|
| 582 |
+
device = obs.device
|
| 583 |
+
t_emb = self._text_cond(texts, text_emb, b, device)
|
| 584 |
+
skips, _ = self.encoder(obs, t_emb)
|
| 585 |
+
return self.change_decoder(skips, t_emb)
|
| 586 |
+
|
| 587 |
+
@torch.no_grad()
|
| 588 |
+
def reconstruct_observation(
|
| 589 |
+
self,
|
| 590 |
+
obs: torch.Tensor,
|
| 591 |
+
text_emb: Optional[torch.Tensor] = None,
|
| 592 |
+
texts: Optional[Sequence[str]] = None,
|
| 593 |
+
) -> torch.Tensor:
|
| 594 |
+
if self.self_decoder is None:
|
| 595 |
+
raise RuntimeError("self_decoder is not initialized. Pass use_self_decoder=True when loading model.")
|
| 596 |
+
b = obs.shape[0]
|
| 597 |
+
device = obs.device
|
| 598 |
+
t_emb = self._text_cond(texts, text_emb, b, device)
|
| 599 |
+
skips, _ = self.encoder(obs, t_emb)
|
| 600 |
+
return self.self_decoder(skips, t_emb)
|
| 601 |
+
|
| 602 |
+
@classmethod
|
| 603 |
+
def from_pretrained(
|
| 604 |
+
cls,
|
| 605 |
+
pretrained_path: Union[str, Path] = ".",
|
| 606 |
+
load_auxiliary: bool = False,
|
| 607 |
+
load_text_encoder: bool = True,
|
| 608 |
+
device: Union[str, torch.device] = "cpu",
|
| 609 |
+
) -> "ParvulusModel":
|
| 610 |
+
root = Path(pretrained_path)
|
| 611 |
+
cfg_file = root / "config.json"
|
| 612 |
+
if cfg_file.exists():
|
| 613 |
+
with open(cfg_file, "r") as f:
|
| 614 |
+
cfg = json.load(f)
|
| 615 |
+
else:
|
| 616 |
+
cfg = {}
|
| 617 |
+
|
| 618 |
+
text_enc = None
|
| 619 |
+
if load_text_encoder:
|
| 620 |
+
from text_encoder import ParvulusTextEncoder
|
| 621 |
+
t5_cfg = cfg.get("text_encoder", {})
|
| 622 |
+
text_enc = ParvulusTextEncoder(
|
| 623 |
+
cond_dim=t5_cfg.get("cond_dim", 256),
|
| 624 |
+
model_name=t5_cfg.get("model_name", "google-t5/t5-small"),
|
| 625 |
+
revision=t5_cfg.get("revision", "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"),
|
| 626 |
+
)
|
| 627 |
+
|
| 628 |
+
model = cls(
|
| 629 |
+
base_channels=cfg.get("base_channels", 48),
|
| 630 |
+
channel_mults=cfg.get("channel_mults", (1, 2, 2, 4)),
|
| 631 |
+
num_res_blocks=cfg.get("num_res_blocks", 1),
|
| 632 |
+
cond_dim=cfg.get("cond_dim", 256),
|
| 633 |
+
action_dim=cfg.get("action_dim", 7),
|
| 634 |
+
chunk_size=cfg.get("chunk_size", 8),
|
| 635 |
+
text_encoder=text_enc,
|
| 636 |
+
use_video_decoder=load_auxiliary,
|
| 637 |
+
use_self_decoder=load_auxiliary,
|
| 638 |
+
use_change_decoder=load_auxiliary,
|
| 639 |
+
)
|
| 640 |
+
|
| 641 |
+
if load_auxiliary and (root / "parvulus_full.safetensors").exists():
|
| 642 |
+
state = load_file(str(root / "parvulus_full.safetensors"))
|
| 643 |
+
model.load_state_dict(state, strict=False)
|
| 644 |
+
else:
|
| 645 |
+
core_weights_file = root / "parvulus.safetensors"
|
| 646 |
+
if not core_weights_file.exists():
|
| 647 |
+
core_weights_file = root / "model.safetensors"
|
| 648 |
+
if not core_weights_file.exists():
|
| 649 |
+
raise FileNotFoundError(f"Neither parvulus.safetensors nor model.safetensors found in {root}")
|
| 650 |
+
|
| 651 |
+
state = load_file(str(core_weights_file))
|
| 652 |
+
model.load_state_dict(state, strict=False)
|
| 653 |
+
|
| 654 |
+
if load_auxiliary:
|
| 655 |
+
aux_weights_file = root / "parvulus_auxiliary.safetensors"
|
| 656 |
+
if aux_weights_file.exists():
|
| 657 |
+
aux_state = load_file(str(aux_weights_file))
|
| 658 |
+
model.load_state_dict(aux_state, strict=False)
|
| 659 |
+
else:
|
| 660 |
+
print(f"[Warning] Auxiliary weights requested but {aux_weights_file} was not found.")
|
| 661 |
+
|
| 662 |
+
model = model.to(device)
|
| 663 |
+
model.eval()
|
| 664 |
+
return model
|
parvulus.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:afb24ea7ba640e5df4ea38ca52f33940bfe6d80b28f7263d2fc8132c60e9969d
|
| 3 |
+
size 39910428
|
parvulus_auxiliary.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c8b83c61918a840f8e3351b465f255c59b2911566ff1a675a53aeb2d4a71496b
|
| 3 |
+
size 116628272
|
parvulus_full.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e420bac1b753214dc86a7694cf53f6f825779d631f0d644be3809d414415ef86
|
| 3 |
+
size 156538852
|
parvulus_runtime.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Standalone high-performance policy runtime for Parvulus via native C ABI ctypes."""
|
| 2 |
+
|
| 3 |
+
import ctypes
|
| 4 |
+
import os
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
from typing import Dict, List, Optional, Sequence, Union
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
_MODULE_DIR = Path(__file__).resolve().parent
|
| 11 |
+
_C_ENGINE_DIR = _MODULE_DIR / "c_engine"
|
| 12 |
+
_LIB_PATH = _C_ENGINE_DIR / "libparvulus.so"
|
| 13 |
+
|
| 14 |
+
_lib: Optional[ctypes.CDLL] = None
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def _load_c_abi() -> ctypes.CDLL:
|
| 18 |
+
global _lib
|
| 19 |
+
if _lib is not None:
|
| 20 |
+
return _lib
|
| 21 |
+
|
| 22 |
+
if not _LIB_PATH.exists():
|
| 23 |
+
try:
|
| 24 |
+
import subprocess
|
| 25 |
+
print(f"[ParvulusRuntime] Building {_LIB_PATH.name}...")
|
| 26 |
+
subprocess.run(["make", "libparvulus.so"], cwd=str(_C_ENGINE_DIR), check=True)
|
| 27 |
+
except Exception:
|
| 28 |
+
pass
|
| 29 |
+
|
| 30 |
+
if not _LIB_PATH.exists():
|
| 31 |
+
raise FileNotFoundError(
|
| 32 |
+
f"Pure CUDA shared library not found at {_LIB_PATH}. "
|
| 33 |
+
f"Run 'make' or './build.sh' inside {_C_ENGINE_DIR} first."
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
lib = ctypes.CDLL(str(_LIB_PATH))
|
| 37 |
+
|
| 38 |
+
lib.parvulus_create.argtypes = [ctypes.c_int]
|
| 39 |
+
lib.parvulus_create.restype = ctypes.c_void_p
|
| 40 |
+
|
| 41 |
+
lib.parvulus_load.argtypes = [
|
| 42 |
+
ctypes.c_void_p,
|
| 43 |
+
ctypes.c_char_p,
|
| 44 |
+
ctypes.c_char_p,
|
| 45 |
+
ctypes.c_char_p,
|
| 46 |
+
]
|
| 47 |
+
lib.parvulus_load.restype = ctypes.c_bool
|
| 48 |
+
|
| 49 |
+
lib.parvulus_set_task_vector.argtypes = [
|
| 50 |
+
ctypes.c_void_p,
|
| 51 |
+
ctypes.POINTER(ctypes.c_float),
|
| 52 |
+
]
|
| 53 |
+
lib.parvulus_set_task_vector.restype = ctypes.c_bool
|
| 54 |
+
|
| 55 |
+
lib.parvulus_set_task_string.argtypes = [
|
| 56 |
+
ctypes.c_void_p,
|
| 57 |
+
ctypes.c_char_p,
|
| 58 |
+
]
|
| 59 |
+
lib.parvulus_set_task_string.restype = ctypes.c_bool
|
| 60 |
+
|
| 61 |
+
lib.parvulus_predict_chunk.argtypes = [
|
| 62 |
+
ctypes.c_void_p,
|
| 63 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 64 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 65 |
+
ctypes.c_char_p,
|
| 66 |
+
ctypes.POINTER(ctypes.c_float),
|
| 67 |
+
ctypes.c_int,
|
| 68 |
+
ctypes.POINTER(ctypes.c_float),
|
| 69 |
+
]
|
| 70 |
+
lib.parvulus_predict_chunk.restype = ctypes.c_bool
|
| 71 |
+
|
| 72 |
+
lib.parvulus_predict_chunk_with_vector.argtypes = [
|
| 73 |
+
ctypes.c_void_p,
|
| 74 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 75 |
+
ctypes.POINTER(ctypes.c_uint8),
|
| 76 |
+
ctypes.POINTER(ctypes.c_float),
|
| 77 |
+
ctypes.POINTER(ctypes.c_float),
|
| 78 |
+
ctypes.c_int,
|
| 79 |
+
ctypes.POINTER(ctypes.c_float),
|
| 80 |
+
]
|
| 81 |
+
lib.parvulus_predict_chunk_with_vector.restype = ctypes.c_bool
|
| 82 |
+
|
| 83 |
+
lib.parvulus_register_task_embedding.argtypes = [
|
| 84 |
+
ctypes.c_void_p,
|
| 85 |
+
ctypes.c_char_p,
|
| 86 |
+
ctypes.POINTER(ctypes.c_float),
|
| 87 |
+
]
|
| 88 |
+
lib.parvulus_register_task_embedding.restype = ctypes.c_bool
|
| 89 |
+
|
| 90 |
+
lib.parvulus_destroy.argtypes = [ctypes.c_void_p]
|
| 91 |
+
lib.parvulus_destroy.restype = None
|
| 92 |
+
|
| 93 |
+
_lib = lib
|
| 94 |
+
return _lib
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
class ParvulusRuntime:
|
| 98 |
+
"""Standalone high-performance Parvulus policy inference engine (Vector-First)."""
|
| 99 |
+
|
| 100 |
+
def __init__(
|
| 101 |
+
self,
|
| 102 |
+
device_id: int = 0,
|
| 103 |
+
weights_path: Optional[Union[str, Path]] = None,
|
| 104 |
+
embeddings_path: Optional[Union[str, Path]] = None,
|
| 105 |
+
scale_path: Optional[Union[str, Path]] = None,
|
| 106 |
+
chunk_size: int = 8,
|
| 107 |
+
replan_stride: int = 8,
|
| 108 |
+
num_euler_steps: int = 10,
|
| 109 |
+
):
|
| 110 |
+
self.device_id = device_id
|
| 111 |
+
self.chunk_size = chunk_size
|
| 112 |
+
self.replan_stride = replan_stride
|
| 113 |
+
self.num_euler_steps = num_euler_steps
|
| 114 |
+
|
| 115 |
+
if weights_path is None:
|
| 116 |
+
weights_path = _C_ENGINE_DIR / "parvulus_model_weights.bin"
|
| 117 |
+
if embeddings_path is None:
|
| 118 |
+
embeddings_path = _C_ENGINE_DIR / "task_embeddings.bin"
|
| 119 |
+
if scale_path is None:
|
| 120 |
+
scale_path = _C_ENGINE_DIR / "action_scale.bin"
|
| 121 |
+
|
| 122 |
+
self.weights_path = Path(weights_path)
|
| 123 |
+
self.embeddings_path = Path(embeddings_path) if embeddings_path else None
|
| 124 |
+
self.scale_path = Path(scale_path) if scale_path else None
|
| 125 |
+
|
| 126 |
+
if not self.weights_path.exists():
|
| 127 |
+
print(f"[ParvulusRuntime] Generating {self.weights_path.name} from SafeTensors...")
|
| 128 |
+
import sys
|
| 129 |
+
if str(_C_ENGINE_DIR) not in sys.path:
|
| 130 |
+
sys.path.insert(0, str(_C_ENGINE_DIR))
|
| 131 |
+
from export_from_safetensors import export_binary_weights, export_action_scale, export_task_embeddings
|
| 132 |
+
|
| 133 |
+
st_path = _MODULE_DIR / "parvulus.safetensors"
|
| 134 |
+
if not st_path.exists():
|
| 135 |
+
st_path = _MODULE_DIR / "model.safetensors"
|
| 136 |
+
export_binary_weights(str(st_path), str(self.weights_path))
|
| 137 |
+
if self.scale_path and not self.scale_path.exists():
|
| 138 |
+
export_action_scale(str(_MODULE_DIR / "action_scale.json"), str(self.scale_path))
|
| 139 |
+
if self.embeddings_path and not self.embeddings_path.exists() and (_C_ENGINE_DIR / "canonical_tasks.json").exists():
|
| 140 |
+
export_task_embeddings(str(_C_ENGINE_DIR / "canonical_tasks.json"), str(st_path), str(self.embeddings_path))
|
| 141 |
+
|
| 142 |
+
self._c_lib = _load_c_abi()
|
| 143 |
+
self.handle = self._c_lib.parvulus_create(self.device_id)
|
| 144 |
+
if not self.handle:
|
| 145 |
+
raise RuntimeError(f"Failed to create Parvulus C/CUDA engine on GPU {self.device_id}")
|
| 146 |
+
|
| 147 |
+
emb_c = str(self.embeddings_path).encode("utf-8") if (self.embeddings_path and self.embeddings_path.exists()) else None
|
| 148 |
+
scl_c = str(self.scale_path).encode("utf-8") if (self.scale_path and self.scale_path.exists()) else None
|
| 149 |
+
|
| 150 |
+
ok = self._c_lib.parvulus_load(
|
| 151 |
+
self.handle,
|
| 152 |
+
str(self.weights_path).encode("utf-8"),
|
| 153 |
+
emb_c,
|
| 154 |
+
scl_c,
|
| 155 |
+
)
|
| 156 |
+
if not ok:
|
| 157 |
+
self._c_lib.parvulus_destroy(self.handle)
|
| 158 |
+
self.handle = None
|
| 159 |
+
raise RuntimeError("Failed to load Parvulus model weights into GPU VRAM!")
|
| 160 |
+
|
| 161 |
+
self.current_task_desc: Optional[str] = None
|
| 162 |
+
self.current_task_vector: Optional[np.ndarray] = None
|
| 163 |
+
self.action_queue: List[np.ndarray] = []
|
| 164 |
+
self.stride_counter: int = 0
|
| 165 |
+
self._actions_buffer = np.zeros(56, dtype=np.float32)
|
| 166 |
+
|
| 167 |
+
self._onnx_session = None
|
| 168 |
+
self._tokenizer = None
|
| 169 |
+
self._torch_text_encoder = None
|
| 170 |
+
|
| 171 |
+
def __del__(self):
|
| 172 |
+
if hasattr(self, "handle") and self.handle is not None and self._c_lib is not None:
|
| 173 |
+
self._c_lib.parvulus_destroy(self.handle)
|
| 174 |
+
self.handle = None
|
| 175 |
+
|
| 176 |
+
def set_task_vector(self, task_vector: Union[np.ndarray, Sequence[float]]) -> bool:
|
| 177 |
+
"""Sets the active 256-dim task conditioning vector on the GPU."""
|
| 178 |
+
vec_c = np.ascontiguousarray(task_vector, dtype=np.float32).flatten()[:256]
|
| 179 |
+
vec_ptr = vec_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 180 |
+
ok = bool(self._c_lib.parvulus_set_task_vector(self.handle, vec_ptr))
|
| 181 |
+
if ok:
|
| 182 |
+
self.current_task_vector = vec_c
|
| 183 |
+
return ok
|
| 184 |
+
|
| 185 |
+
def reset(
|
| 186 |
+
self,
|
| 187 |
+
task_description: Optional[str] = None,
|
| 188 |
+
task_vector: Optional[Union[np.ndarray, Sequence[float]]] = None,
|
| 189 |
+
):
|
| 190 |
+
"""Resets action queue and updates task conditioning for a new episode.
|
| 191 |
+
|
| 192 |
+
Accepts:
|
| 193 |
+
- task_vector: 256-dim float vector (first-class input).
|
| 194 |
+
- task_description: text instruction string (automatically handles perturbations
|
| 195 |
+
by encoding them on the fly into 256-dim conditioning vectors).
|
| 196 |
+
"""
|
| 197 |
+
if task_vector is not None:
|
| 198 |
+
self.set_task_vector(task_vector)
|
| 199 |
+
self.current_task_desc = task_description or "<custom_vector>"
|
| 200 |
+
elif task_description is not None:
|
| 201 |
+
self.current_task_desc = task_description
|
| 202 |
+
task_bytes = task_description.encode("utf-8")
|
| 203 |
+
if not self._c_lib.parvulus_set_task_string(self.handle, task_bytes):
|
| 204 |
+
encoded_vec = self.encode_text(task_description)
|
| 205 |
+
self.set_task_vector(encoded_vec)
|
| 206 |
+
|
| 207 |
+
self.action_queue.clear()
|
| 208 |
+
self.stride_counter = 0
|
| 209 |
+
|
| 210 |
+
def register_task_embedding(self, task_description: str, text_embedding: np.ndarray) -> bool:
|
| 211 |
+
"""Registers a 256-dim embedding vector for a task description in engine memory."""
|
| 212 |
+
task_bytes = task_description.encode("utf-8")
|
| 213 |
+
emb_c = np.ascontiguousarray(text_embedding, dtype=np.float32).flatten()[:256]
|
| 214 |
+
emb_ptr = emb_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 215 |
+
return bool(self._c_lib.parvulus_register_task_embedding(self.handle, task_bytes, emb_ptr))
|
| 216 |
+
|
| 217 |
+
def encode_text(self, text: str) -> np.ndarray:
|
| 218 |
+
"""Encodes an arbitrary natural language prompt into a 256-dim vector."""
|
| 219 |
+
onnx_path = _C_ENGINE_DIR / "parvulus_text_encoder.onnx"
|
| 220 |
+
if not onnx_path.exists() and os.environ.get("PARVULUS_HF_REPO"):
|
| 221 |
+
try:
|
| 222 |
+
from huggingface_hub import hf_hub_download
|
| 223 |
+
hf_hub_download(
|
| 224 |
+
repo_id=os.environ["PARVULUS_HF_REPO"],
|
| 225 |
+
filename="parvulus_text_encoder.onnx",
|
| 226 |
+
subfolder="c_engine",
|
| 227 |
+
local_dir=str(_MODULE_DIR),
|
| 228 |
+
)
|
| 229 |
+
except Exception:
|
| 230 |
+
pass
|
| 231 |
+
|
| 232 |
+
if onnx_path.exists():
|
| 233 |
+
try:
|
| 234 |
+
import onnxruntime as ort
|
| 235 |
+
from transformers import AutoTokenizer
|
| 236 |
+
|
| 237 |
+
if self._onnx_session is None:
|
| 238 |
+
available = ort.get_available_providers()
|
| 239 |
+
providers = [p for p in ["CUDAExecutionProvider", "CPUExecutionProvider"] if p in available]
|
| 240 |
+
self._onnx_session = ort.InferenceSession(str(onnx_path), providers=providers)
|
| 241 |
+
self._tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small")
|
| 242 |
+
|
| 243 |
+
tokens = self._tokenizer([text], padding=True, truncation=True, return_tensors="np")
|
| 244 |
+
ort_out = self._onnx_session.run(
|
| 245 |
+
None,
|
| 246 |
+
{
|
| 247 |
+
"input_ids": tokens["input_ids"].astype(np.int64),
|
| 248 |
+
"attention_mask": tokens["attention_mask"].astype(np.int64),
|
| 249 |
+
},
|
| 250 |
+
)[0]
|
| 251 |
+
return ort_out[0].astype(np.float32)
|
| 252 |
+
except Exception:
|
| 253 |
+
pass
|
| 254 |
+
|
| 255 |
+
try:
|
| 256 |
+
import torch
|
| 257 |
+
from text_encoder import ParvulusTextEncoder
|
| 258 |
+
from safetensors.torch import load_file
|
| 259 |
+
|
| 260 |
+
if self._torch_text_encoder is None:
|
| 261 |
+
te = ParvulusTextEncoder().cpu().eval()
|
| 262 |
+
st_path = _MODULE_DIR / "parvulus.safetensors"
|
| 263 |
+
if st_path.exists():
|
| 264 |
+
weights = load_file(str(st_path))
|
| 265 |
+
proj_dict = {
|
| 266 |
+
"0.weight": weights["text_encoder.proj.0.weight"],
|
| 267 |
+
"0.bias": weights["text_encoder.proj.0.bias"],
|
| 268 |
+
"2.weight": weights["text_encoder.proj.2.weight"],
|
| 269 |
+
"2.bias": weights["text_encoder.proj.2.bias"],
|
| 270 |
+
}
|
| 271 |
+
te.proj.load_state_dict(proj_dict)
|
| 272 |
+
self._torch_text_encoder = te
|
| 273 |
+
|
| 274 |
+
with torch.no_grad():
|
| 275 |
+
emb = self._torch_text_encoder([text]).cpu().numpy()[0]
|
| 276 |
+
return emb.astype(np.float32)
|
| 277 |
+
except Exception as e:
|
| 278 |
+
raise RuntimeError(
|
| 279 |
+
f"Failed to dynamically encode text prompt '{text}': {e}. "
|
| 280 |
+
"Ensure onnxruntime or transformers/torch is installed."
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
def predict_chunk(
|
| 284 |
+
self,
|
| 285 |
+
agentview_img: np.ndarray,
|
| 286 |
+
wrist_img: np.ndarray,
|
| 287 |
+
task_vector: Optional[Union[np.ndarray, Sequence[float]]] = None,
|
| 288 |
+
task_description: Optional[str] = None,
|
| 289 |
+
custom_z0: Optional[np.ndarray] = None,
|
| 290 |
+
) -> np.ndarray:
|
| 291 |
+
"""Predicts an [8, 7] action chunk using active or newly provided task conditioning."""
|
| 292 |
+
if task_vector is not None or task_description is not None:
|
| 293 |
+
self.reset(task_description=task_description, task_vector=task_vector)
|
| 294 |
+
|
| 295 |
+
agent_c = np.ascontiguousarray(agentview_img, dtype=np.uint8)
|
| 296 |
+
wrist_c = np.ascontiguousarray(wrist_img, dtype=np.uint8)
|
| 297 |
+
|
| 298 |
+
agent_ptr = agent_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
|
| 299 |
+
wrist_ptr = wrist_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
|
| 300 |
+
out_ptr = self._actions_buffer.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 301 |
+
|
| 302 |
+
z0_ptr = None
|
| 303 |
+
if custom_z0 is not None:
|
| 304 |
+
z0_c = np.ascontiguousarray(custom_z0, dtype=np.float32)
|
| 305 |
+
z0_ptr = z0_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
|
| 306 |
+
|
| 307 |
+
ok = self._c_lib.parvulus_predict_chunk(
|
| 308 |
+
self.handle,
|
| 309 |
+
agent_ptr,
|
| 310 |
+
wrist_ptr,
|
| 311 |
+
None,
|
| 312 |
+
out_ptr,
|
| 313 |
+
self.num_euler_steps,
|
| 314 |
+
z0_ptr,
|
| 315 |
+
)
|
| 316 |
+
if not ok:
|
| 317 |
+
raise RuntimeError("Pure CUDA predict_chunk failed! Check task conditioning vector.")
|
| 318 |
+
|
| 319 |
+
return self._actions_buffer.reshape(8, 7).copy()
|
| 320 |
+
|
| 321 |
+
def predict_chunk_with_emb(
|
| 322 |
+
self,
|
| 323 |
+
agentview_img: np.ndarray,
|
| 324 |
+
wrist_img: np.ndarray,
|
| 325 |
+
text_embedding: np.ndarray,
|
| 326 |
+
custom_z0: Optional[np.ndarray] = None,
|
| 327 |
+
) -> np.ndarray:
|
| 328 |
+
"""Alias for predict_chunk(agentview_img, wrist_img, task_vector=text_embedding)."""
|
| 329 |
+
return self.predict_chunk(
|
| 330 |
+
agentview_img=agentview_img,
|
| 331 |
+
wrist_img=wrist_img,
|
| 332 |
+
task_vector=text_embedding,
|
| 333 |
+
custom_z0=custom_z0,
|
| 334 |
+
)
|
| 335 |
+
|
| 336 |
+
def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray:
|
| 337 |
+
"""Executes a single 7-DoF step, automatically managing the receding horizon action queue."""
|
| 338 |
+
if not self.action_queue or self.stride_counter >= self.replan_stride:
|
| 339 |
+
chunk = self.predict_chunk(agentview_img, wrist_img)
|
| 340 |
+
self.action_queue = list(chunk[: self.replan_stride])
|
| 341 |
+
self.stride_counter = 0
|
| 342 |
+
|
| 343 |
+
action = self.action_queue.pop(0)
|
| 344 |
+
self.stride_counter += 1
|
| 345 |
+
return action
|
prepare_runtime.py
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Preparation and setup script for Parvulus standalone native runtime assets."""
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
import os
|
| 6 |
+
import subprocess
|
| 7 |
+
import sys
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
ROOT_DIR = Path(__file__).resolve().parent
|
| 11 |
+
C_ENGINE_DIR = ROOT_DIR / "c_engine"
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def build_c_engine():
|
| 15 |
+
"""Compiles C++/CUDA engine using make."""
|
| 16 |
+
lib_path = C_ENGINE_DIR / "libparvulus.so"
|
| 17 |
+
cli_path = C_ENGINE_DIR / "parvulus_runtime"
|
| 18 |
+
if lib_path.exists() and cli_path.exists():
|
| 19 |
+
print(f"[Prepare] Found existing {lib_path.name} and {cli_path.name}.")
|
| 20 |
+
return
|
| 21 |
+
|
| 22 |
+
print("\n[Prepare] Building C++/CUDA engine...")
|
| 23 |
+
res = subprocess.run(["make", "-j", str(os.cpu_count() or 4)], cwd=str(C_ENGINE_DIR))
|
| 24 |
+
if res.returncode != 0:
|
| 25 |
+
raise RuntimeError(f"C engine build failed with exit code {res.returncode}")
|
| 26 |
+
print("[Prepare] C engine compiled successfully.")
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def export_binary_assets():
|
| 30 |
+
"""Exports parvulus_model_weights.bin, action_scale.bin, and task_embeddings.bin from SafeTensors/JSON."""
|
| 31 |
+
out_bin = C_ENGINE_DIR / "parvulus_model_weights.bin"
|
| 32 |
+
out_scale = C_ENGINE_DIR / "action_scale.bin"
|
| 33 |
+
out_emb = C_ENGINE_DIR / "task_embeddings.bin"
|
| 34 |
+
|
| 35 |
+
st_path = ROOT_DIR / "parvulus.safetensors"
|
| 36 |
+
if not st_path.exists():
|
| 37 |
+
st_path = ROOT_DIR / "model.safetensors"
|
| 38 |
+
if not st_path.exists():
|
| 39 |
+
raise FileNotFoundError(f"SafeTensors weights not found in {ROOT_DIR}")
|
| 40 |
+
|
| 41 |
+
sys.path.insert(0, str(C_ENGINE_DIR))
|
| 42 |
+
from export_from_safetensors import export_binary_weights, export_action_scale, export_task_embeddings
|
| 43 |
+
|
| 44 |
+
if not out_bin.exists():
|
| 45 |
+
print(f"\n[Prepare] Exporting binary model weights from {st_path.name}...")
|
| 46 |
+
export_binary_weights(str(st_path), str(out_bin))
|
| 47 |
+
else:
|
| 48 |
+
print(f"[Prepare] Found existing {out_bin.name} ({out_bin.stat().st_size / (1024*1024):.2f} MB).")
|
| 49 |
+
|
| 50 |
+
if not out_scale.exists():
|
| 51 |
+
export_action_scale(str(ROOT_DIR / "action_scale.json"), str(out_scale))
|
| 52 |
+
else:
|
| 53 |
+
print(f"[Prepare] Found existing {out_scale.name}.")
|
| 54 |
+
|
| 55 |
+
if not out_emb.exists() and (C_ENGINE_DIR / "canonical_tasks.json").exists():
|
| 56 |
+
export_task_embeddings(str(C_ENGINE_DIR / "canonical_tasks.json"), str(st_path), str(out_emb))
|
| 57 |
+
elif out_emb.exists():
|
| 58 |
+
print(f"[Prepare] Found existing {out_emb.name}.")
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def obtain_text_encoder_onnx(hf_repo: str = None, force_convert: bool = False):
|
| 62 |
+
"""Obtains parvulus_text_encoder.onnx via Hugging Face download or local conversion."""
|
| 63 |
+
onnx_path = C_ENGINE_DIR / "parvulus_text_encoder.onnx"
|
| 64 |
+
if onnx_path.exists() and not force_convert:
|
| 65 |
+
print(f"[Prepare] Found existing {onnx_path.name} ({onnx_path.stat().st_size / (1024*1024):.1f} MB).")
|
| 66 |
+
return
|
| 67 |
+
|
| 68 |
+
repo_id = hf_repo or os.environ.get("PARVULUS_HF_REPO")
|
| 69 |
+
if repo_id and not force_convert:
|
| 70 |
+
print(f"\n[Prepare] Attempting to download {onnx_path.name} from Hugging Face Hub: '{repo_id}'...")
|
| 71 |
+
try:
|
| 72 |
+
from huggingface_hub import hf_hub_download
|
| 73 |
+
|
| 74 |
+
downloaded_file = hf_hub_download(
|
| 75 |
+
repo_id=repo_id,
|
| 76 |
+
filename="parvulus_text_encoder.onnx",
|
| 77 |
+
subfolder="c_engine",
|
| 78 |
+
local_dir=str(ROOT_DIR),
|
| 79 |
+
)
|
| 80 |
+
if Path(downloaded_file).exists():
|
| 81 |
+
print(f"[Prepare] Successfully downloaded ONNX model from Hugging Face Hub.")
|
| 82 |
+
return
|
| 83 |
+
except Exception as e:
|
| 84 |
+
print(f"[Prepare] Hugging Face download from '{repo_id}' failed: {e}.")
|
| 85 |
+
print("[Prepare] Falling back to local conversion...")
|
| 86 |
+
|
| 87 |
+
print(f"\n[Prepare] Converting text encoder to ONNX locally (c_engine/export_text_encoder_onnx.py)...")
|
| 88 |
+
sys.path.insert(0, str(C_ENGINE_DIR))
|
| 89 |
+
from export_text_encoder_onnx import export_onnx
|
| 90 |
+
|
| 91 |
+
export_onnx(
|
| 92 |
+
weights_path=str(ROOT_DIR / "parvulus.safetensors"),
|
| 93 |
+
output_path=str(onnx_path),
|
| 94 |
+
verify=True,
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def main():
|
| 99 |
+
parser = argparse.ArgumentParser(description="Prepare PARVULUS Standalone Native Runtime.")
|
| 100 |
+
parser.add_argument(
|
| 101 |
+
"--hf_repo",
|
| 102 |
+
type=str,
|
| 103 |
+
default=None,
|
| 104 |
+
help="Optional Hugging Face repository ID (e.g. 'username/parvulus') to download pre-exported ONNX from",
|
| 105 |
+
)
|
| 106 |
+
parser.add_argument(
|
| 107 |
+
"--build_c",
|
| 108 |
+
action="store_true",
|
| 109 |
+
help="Force rebuild of C++/CUDA engine",
|
| 110 |
+
)
|
| 111 |
+
parser.add_argument(
|
| 112 |
+
"--convert_onnx",
|
| 113 |
+
action="store_true",
|
| 114 |
+
help="Force local ONNX conversion instead of downloading from HF",
|
| 115 |
+
)
|
| 116 |
+
parser.add_argument(
|
| 117 |
+
"--skip_onnx",
|
| 118 |
+
action="store_true",
|
| 119 |
+
help="Skip ONNX text encoder preparation (rely on pure vector conditioning or PyTorch fallback)",
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
args = parser.parse_args()
|
| 123 |
+
|
| 124 |
+
print("=" * 70)
|
| 125 |
+
print(" PREPARING PARVULUS STANDALONE RUNTIME")
|
| 126 |
+
print("=" * 70)
|
| 127 |
+
|
| 128 |
+
if args.build_c:
|
| 129 |
+
res = subprocess.run(["make", "clean"], cwd=str(C_ENGINE_DIR))
|
| 130 |
+
build_c_engine()
|
| 131 |
+
export_binary_assets()
|
| 132 |
+
|
| 133 |
+
if not args.skip_onnx:
|
| 134 |
+
obtain_text_encoder_onnx(hf_repo=args.hf_repo, force_convert=args.convert_onnx)
|
| 135 |
+
else:
|
| 136 |
+
print("[Prepare] Skipping ONNX preparation (--skip_onnx set).")
|
| 137 |
+
|
| 138 |
+
print("\n" + "=" * 70)
|
| 139 |
+
print(" PARVULUS STANDALONE RUNTIME IS READY FOR DEPLOYMENT!")
|
| 140 |
+
print("=" * 70)
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
if __name__ == "__main__":
|
| 144 |
+
main()
|
pyproject.toml
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[build-system]
|
| 2 |
+
requires = ["setuptools>=61.0"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
+
|
| 5 |
+
[project]
|
| 6 |
+
name = "parvulus"
|
| 7 |
+
version = "1.0.0"
|
| 8 |
+
description = "PARVULUS: 10M-Scale World Action Model for Real-Time Robot Manipulation"
|
| 9 |
+
readme = "README.md"
|
| 10 |
+
requires-python = ">=3.8"
|
| 11 |
+
license = {text = "Apache-2.0"}
|
| 12 |
+
authors = [
|
| 13 |
+
{name = "Parvulus Team"}
|
| 14 |
+
]
|
| 15 |
+
dependencies = [
|
| 16 |
+
"torch>=2.2.0",
|
| 17 |
+
"torchvision>=0.17.0",
|
| 18 |
+
"transformers>=4.40.0",
|
| 19 |
+
"numpy>=1.24.0,<2.3.0",
|
| 20 |
+
"Pillow>=10.0.0",
|
| 21 |
+
"opencv-python>=4.8.0",
|
| 22 |
+
"safetensors>=0.4.0",
|
| 23 |
+
"huggingface_hub>=0.20.0",
|
| 24 |
+
"onnxruntime>=1.16.0",
|
| 25 |
+
]
|
| 26 |
+
|
| 27 |
+
[project.optional-dependencies]
|
| 28 |
+
eval = [
|
| 29 |
+
"robosuite>=1.4.1",
|
| 30 |
+
"gymnasium>=0.29.0",
|
| 31 |
+
"h5py>=3.10.0",
|
| 32 |
+
]
|
| 33 |
+
|
| 34 |
+
[tool.setuptools]
|
| 35 |
+
py-modules = [
|
| 36 |
+
"parvulus",
|
| 37 |
+
"infer",
|
| 38 |
+
"parvulus_runtime",
|
| 39 |
+
"text_encoder",
|
| 40 |
+
"action_utils",
|
| 41 |
+
"prepare_runtime"
|
| 42 |
+
]
|
requirements.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.2.0
|
| 2 |
+
torchvision>=0.17.0
|
| 3 |
+
transformers>=4.40.0
|
| 4 |
+
numpy>=1.24.0,<2.3.0
|
| 5 |
+
Pillow>=10.0.0
|
| 6 |
+
opencv-python>=4.8.0
|
| 7 |
+
safetensors>=0.4.0
|
| 8 |
+
huggingface_hub>=0.20.0
|
| 9 |
+
onnxruntime>=1.16.0
|
requirements_eval.txt
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
torch>=2.2.0
|
| 2 |
+
torchvision>=0.17.0
|
| 3 |
+
transformers>=4.40.0
|
| 4 |
+
numpy>=1.24.0,<2.3.0
|
| 5 |
+
scipy>=1.11.0
|
| 6 |
+
h5py>=3.10.0
|
| 7 |
+
opencv-python>=4.8.0
|
| 8 |
+
imageio>=2.30.0
|
| 9 |
+
imageio-ffmpeg>=0.4.8
|
| 10 |
+
robosuite==1.4.1
|
| 11 |
+
mujoco>=2.3.7
|
| 12 |
+
easydict
|
| 13 |
+
pyyaml
|
| 14 |
+
huggingface_hub>=0.20.0
|
| 15 |
+
safetensors>=0.4.0
|
test_continuous_pipelined.py
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Continuous pipelined execution benchmark for Parvulus comparing sequential vs overlapped inference."""
|
| 3 |
+
|
| 4 |
+
import argparse
|
| 5 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 6 |
+
import os
|
| 7 |
+
import sys
|
| 8 |
+
import time
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
from typing import Dict, List, Tuple
|
| 11 |
+
|
| 12 |
+
import numpy as np
|
| 13 |
+
import torch
|
| 14 |
+
|
| 15 |
+
PROJECT_ROOT = Path(__file__).resolve().parent
|
| 16 |
+
if str(PROJECT_ROOT) not in sys.path:
|
| 17 |
+
sys.path.insert(0, str(PROJECT_ROOT))
|
| 18 |
+
|
| 19 |
+
LIBERO_ROOT = Path("/home/ubuntu/libero-difficulty/LIBERO")
|
| 20 |
+
if str(LIBERO_ROOT) not in sys.path:
|
| 21 |
+
sys.path.insert(0, str(LIBERO_ROOT))
|
| 22 |
+
|
| 23 |
+
from libero.libero import benchmark
|
| 24 |
+
from libero.libero.envs import OffScreenRenderEnv
|
| 25 |
+
from libero.libero import get_libero_path
|
| 26 |
+
|
| 27 |
+
from infer import ParvulusInference
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def evaluate_trial(
|
| 31 |
+
env: OffScreenRenderEnv,
|
| 32 |
+
pipeline: ParvulusInference,
|
| 33 |
+
task_desc: str,
|
| 34 |
+
init_state: np.ndarray,
|
| 35 |
+
seed: int,
|
| 36 |
+
mode: str = "sequential", # "sequential" or "pipelined"
|
| 37 |
+
max_steps: int = 500,
|
| 38 |
+
) -> Dict[str, float]:
|
| 39 |
+
"""Runs a single rollout in sequential or continuous pipelined mode."""
|
| 40 |
+
env.seed(seed)
|
| 41 |
+
torch.manual_seed(seed)
|
| 42 |
+
np.random.seed(seed)
|
| 43 |
+
|
| 44 |
+
obs = env.reset()
|
| 45 |
+
obs = env.set_init_state(init_state)
|
| 46 |
+
pipeline.reset(task_desc)
|
| 47 |
+
|
| 48 |
+
settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
|
| 49 |
+
for _ in range(5):
|
| 50 |
+
obs, _, _, _ = env.step(settle_noop)
|
| 51 |
+
|
| 52 |
+
done = False
|
| 53 |
+
step_count = 0
|
| 54 |
+
total_robot_idle_time = 0.0
|
| 55 |
+
num_replans = 0
|
| 56 |
+
|
| 57 |
+
wall_start = time.perf_counter()
|
| 58 |
+
|
| 59 |
+
if mode == "sequential":
|
| 60 |
+
pipeline.replan_stride = 8
|
| 61 |
+
while not done and step_count < max_steps:
|
| 62 |
+
t0 = time.perf_counter()
|
| 63 |
+
action = pipeline.step(obs["agentview_image"], obs["robot0_eye_in_hand_image"])
|
| 64 |
+
wait_time = time.perf_counter() - t0
|
| 65 |
+
if wait_time > 0.005:
|
| 66 |
+
total_robot_idle_time += wait_time
|
| 67 |
+
num_replans += 1
|
| 68 |
+
|
| 69 |
+
obs, reward, done, info = env.step(action)
|
| 70 |
+
step_count += 1
|
| 71 |
+
if done:
|
| 72 |
+
break
|
| 73 |
+
|
| 74 |
+
elif mode == "pipelined":
|
| 75 |
+
t_init = time.perf_counter()
|
| 76 |
+
agent_img = obs["agentview_image"]
|
| 77 |
+
wrist_img = obs["robot0_eye_in_hand_image"]
|
| 78 |
+
current_chunk = pipeline.predict_chunk(agent_img, wrist_img)
|
| 79 |
+
init_idle = time.perf_counter() - t_init
|
| 80 |
+
total_robot_idle_time += init_idle
|
| 81 |
+
num_replans += 1
|
| 82 |
+
|
| 83 |
+
executor = ThreadPoolExecutor(max_workers=1)
|
| 84 |
+
|
| 85 |
+
for a in current_chunk[0:4]:
|
| 86 |
+
obs, reward, done, _ = env.step(a)
|
| 87 |
+
step_count += 1
|
| 88 |
+
if done:
|
| 89 |
+
break
|
| 90 |
+
|
| 91 |
+
while not done and step_count < max_steps:
|
| 92 |
+
agent_img = obs["agentview_image"]
|
| 93 |
+
wrist_img = obs["robot0_eye_in_hand_image"]
|
| 94 |
+
|
| 95 |
+
future = executor.submit(pipeline.predict_chunk, agent_img, wrist_img)
|
| 96 |
+
|
| 97 |
+
for a in current_chunk[4:8]:
|
| 98 |
+
obs, reward, done, _ = env.step(a)
|
| 99 |
+
step_count += 1
|
| 100 |
+
if done:
|
| 101 |
+
break
|
| 102 |
+
if done:
|
| 103 |
+
break
|
| 104 |
+
|
| 105 |
+
tw0 = time.perf_counter()
|
| 106 |
+
current_chunk = future.result()
|
| 107 |
+
idle_wait = time.perf_counter() - tw0
|
| 108 |
+
total_robot_idle_time += idle_wait
|
| 109 |
+
num_replans += 1
|
| 110 |
+
|
| 111 |
+
for a in current_chunk[0:4]:
|
| 112 |
+
obs, reward, done, _ = env.step(a)
|
| 113 |
+
step_count += 1
|
| 114 |
+
if done:
|
| 115 |
+
break
|
| 116 |
+
|
| 117 |
+
executor.shutdown(wait=False)
|
| 118 |
+
|
| 119 |
+
wall_total = time.perf_counter() - wall_start
|
| 120 |
+
success = float(done)
|
| 121 |
+
fps = step_count / wall_total if wall_total > 0 else 0.0
|
| 122 |
+
|
| 123 |
+
return {
|
| 124 |
+
"success": success,
|
| 125 |
+
"steps": step_count,
|
| 126 |
+
"wall_time_s": wall_total,
|
| 127 |
+
"fps": fps,
|
| 128 |
+
"num_replans": num_replans,
|
| 129 |
+
"total_robot_idle_ms": total_robot_idle_time * 1000.0,
|
| 130 |
+
"avg_idle_per_replan_ms": (total_robot_idle_time * 1000.0) / max(1, num_replans),
|
| 131 |
+
"avg_idle_per_step_ms": (total_robot_idle_time * 1000.0) / max(1, step_count),
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def main():
|
| 136 |
+
parser = argparse.ArgumentParser(description="Continuous Pipelined Benchmark")
|
| 137 |
+
parser.add_argument("--device", type=str, default="cuda:0")
|
| 138 |
+
parser.add_argument("--suite", type=str, default="libero_spatial")
|
| 139 |
+
parser.add_argument("--task-id", type=int, default=0)
|
| 140 |
+
parser.add_argument("--trials", type=int, default=3)
|
| 141 |
+
args = parser.parse_args()
|
| 142 |
+
|
| 143 |
+
os.environ["CUDA_VISIBLE_DEVICES"] = args.device.replace("cuda:", "")
|
| 144 |
+
os.environ["MUJOCO_GL"] = "egl"
|
| 145 |
+
|
| 146 |
+
print("=" * 75)
|
| 147 |
+
print(f" CONTINUOUS PIPELINED EXECUTION BENCHMARK ({args.suite}, Task #{args.task_id})")
|
| 148 |
+
print(f" Device: {args.device} | Trials: {args.trials}")
|
| 149 |
+
print("=" * 75)
|
| 150 |
+
|
| 151 |
+
pipeline = ParvulusInference.from_pretrained(
|
| 152 |
+
PROJECT_ROOT,
|
| 153 |
+
device="cuda:0",
|
| 154 |
+
num_ode_steps=10,
|
| 155 |
+
chunk_size=8,
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
bench_dict = benchmark.get_benchmark_dict()
|
| 159 |
+
suite = bench_dict[args.suite]()
|
| 160 |
+
task_obj = suite.get_task(args.task_id)
|
| 161 |
+
task_desc = task_obj.language
|
| 162 |
+
bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file
|
| 163 |
+
|
| 164 |
+
with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]):
|
| 165 |
+
init_states = suite.get_task_init_states(args.task_id)
|
| 166 |
+
|
| 167 |
+
env = OffScreenRenderEnv(
|
| 168 |
+
bddl_file_name=str(bddl_file),
|
| 169 |
+
camera_heights=256,
|
| 170 |
+
camera_widths=256,
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
modes = ["sequential", "pipelined"]
|
| 174 |
+
results = {m: [] for m in modes}
|
| 175 |
+
|
| 176 |
+
for mode in modes:
|
| 177 |
+
print(f"\nEvaluating mode: {mode.upper()}...")
|
| 178 |
+
for trial in range(args.trials):
|
| 179 |
+
state = init_states[trial]
|
| 180 |
+
seed = 42 + trial
|
| 181 |
+
res = evaluate_trial(env, pipeline, task_desc, state, seed=seed, mode=mode)
|
| 182 |
+
results[mode].append(res)
|
| 183 |
+
print(
|
| 184 |
+
f" Trial {trial+1}/{args.trials}: "
|
| 185 |
+
f"Success={bool(res['success'])} | "
|
| 186 |
+
f"Steps={res['steps']} | "
|
| 187 |
+
f"Wall={res['wall_time_s']:.2f}s | "
|
| 188 |
+
f"Rate={res['fps']:.1f} steps/s | "
|
| 189 |
+
f"Total Idle={res['total_robot_idle_ms']:.2f} ms | "
|
| 190 |
+
f"Idle/Replan={res['avg_idle_per_replan_ms']:.2f} ms"
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
env.close()
|
| 194 |
+
|
| 195 |
+
print("\n" + "=" * 75)
|
| 196 |
+
print(" CONTINUOUS PIPELINED EXECUTION - FINAL COMPARATIVE REPORT:")
|
| 197 |
+
print("=" * 75)
|
| 198 |
+
for mode in modes:
|
| 199 |
+
res_list = results[mode]
|
| 200 |
+
succ = np.mean([r["success"] for r in res_list]) * 100.0
|
| 201 |
+
mean_steps = np.mean([r["steps"] for r in res_list])
|
| 202 |
+
mean_fps = np.mean([r["fps"] for r in res_list])
|
| 203 |
+
mean_idle = np.mean([r["total_robot_idle_ms"] for r in res_list])
|
| 204 |
+
mean_idle_replan = np.mean([r["avg_idle_per_replan_ms"] for r in res_list])
|
| 205 |
+
mean_idle_step = np.mean([r["avg_idle_per_step_ms"] for r in res_list])
|
| 206 |
+
|
| 207 |
+
print(f"Mode: {mode.upper()}")
|
| 208 |
+
print(f" - Benchmark Success Rate: {succ:.1f}%")
|
| 209 |
+
print(f" - Mean Steps to Task Success: {mean_steps:.1f} steps")
|
| 210 |
+
print(f" - Actual Execution Frequency: {mean_fps:.2f} steps/sec")
|
| 211 |
+
print(f" - Robot Idle Wait per Replan: {mean_idle_replan:.2f} ms")
|
| 212 |
+
print(f" - Robot Idle Wait per Step: {mean_idle_step:.3f} ms")
|
| 213 |
+
print(f" - Total Robot Idle per Rollout: {mean_idle:.2f} ms")
|
| 214 |
+
print("=" * 75)
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
if __name__ == "__main__":
|
| 218 |
+
main()
|
test_standalone_runtime.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Comprehensive test script verifying Parvulus standalone runtime and Python bindings (Vector-First & Perturbations)."""
|
| 3 |
+
|
| 4 |
+
import time
|
| 5 |
+
import numpy as np
|
| 6 |
+
from parvulus_runtime import ParvulusRuntime
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def test_vector_and_perturbations():
|
| 10 |
+
print("=" * 75)
|
| 11 |
+
print(" TESTING PARVULUS STANDALONE RUNTIME: VECTOR-FIRST & PERTURBATIONS")
|
| 12 |
+
print("=" * 75)
|
| 13 |
+
|
| 14 |
+
# 1. Initialize Runtime
|
| 15 |
+
print("\n1. Initializing ParvulusRuntime on CUDA:0...")
|
| 16 |
+
t0 = time.perf_counter()
|
| 17 |
+
runtime = ParvulusRuntime(device_id=0, num_euler_steps=10)
|
| 18 |
+
init_ms = (time.perf_counter() - t0) * 1000
|
| 19 |
+
print(f" -> Runtime successfully initialized in {init_ms:.1f} ms.")
|
| 20 |
+
|
| 21 |
+
agent_img = np.zeros((256, 256, 3), dtype=np.uint8)
|
| 22 |
+
wrist_img = np.zeros((256, 256, 3), dtype=np.uint8)
|
| 23 |
+
|
| 24 |
+
# 2. Test First-Class Vector Conditioning
|
| 25 |
+
print("\n2. Testing First-Class Vector Conditioning (task_vector)...")
|
| 26 |
+
synthetic_vector = np.random.randn(256).astype(np.float32)
|
| 27 |
+
synthetic_vector /= np.linalg.norm(synthetic_vector)
|
| 28 |
+
|
| 29 |
+
runtime.reset(task_vector=synthetic_vector)
|
| 30 |
+
chunk_vec = runtime.predict_chunk(agent_img, wrist_img)
|
| 31 |
+
print(f" -> Returned chunk shape: {chunk_vec.shape} (dtype: {chunk_vec.dtype})")
|
| 32 |
+
print(f" -> Step 0 actions: {np.round(chunk_vec[0], 4)}")
|
| 33 |
+
|
| 34 |
+
# 3. Test Canonical Instruction String
|
| 35 |
+
canonical_prompt = "pick up the black bowl between the plate and the ramekin and place it on the plate"
|
| 36 |
+
print(f"\n3. Testing Canonical Prompt: '{canonical_prompt}'...")
|
| 37 |
+
runtime.reset(task_description=canonical_prompt)
|
| 38 |
+
chunk_canon = runtime.predict_chunk(agent_img, wrist_img)
|
| 39 |
+
print(f" -> Step 0 actions: {np.round(chunk_canon[0], 4)}")
|
| 40 |
+
|
| 41 |
+
# 4. Test Language Perturbations (Unseen Variations)
|
| 42 |
+
perturbations = [
|
| 43 |
+
"pick up the black bowl and place it on the plate",
|
| 44 |
+
"grasp the black bowl between the plate and the ramekin and place it on the plate",
|
| 45 |
+
"take the black bowl and set it onto the plate",
|
| 46 |
+
"grab the black bowl and put it on the plate",
|
| 47 |
+
]
|
| 48 |
+
|
| 49 |
+
print("\n4. Testing Arbitrary Language Perturbations (Dynamic Vector Encoding)...")
|
| 50 |
+
for idx, perturbed_prompt in enumerate(perturbations, start=1):
|
| 51 |
+
t0 = time.perf_counter()
|
| 52 |
+
runtime.reset(task_description=perturbed_prompt)
|
| 53 |
+
chunk_pert = runtime.predict_chunk(agent_img, wrist_img)
|
| 54 |
+
dt_ms = (time.perf_counter() - t0) * 1000
|
| 55 |
+
|
| 56 |
+
# Cosine similarity with canonical conditioning
|
| 57 |
+
cos_sim = np.dot(runtime.current_task_vector, runtime.encode_text(canonical_prompt)) / (
|
| 58 |
+
np.linalg.norm(runtime.current_task_vector) * np.linalg.norm(runtime.encode_text(canonical_prompt))
|
| 59 |
+
)
|
| 60 |
+
print(f" [Perturbation {idx}] \"{perturbed_prompt}\"")
|
| 61 |
+
print(f" -> Vector Cosine Similarity to Canonical: {cos_sim:.4f}")
|
| 62 |
+
print(f" -> Predicted Action [0]: {np.round(chunk_pert[0], 4)} (Reset+Infer: {dt_ms:.1f} ms)")
|
| 63 |
+
|
| 64 |
+
# 5. Test Step-by-Step Execution with Active Conditioning Vector
|
| 65 |
+
print("\n5. Testing step() Receding Horizon with Active Task Vector across 16 steps...")
|
| 66 |
+
runtime.reset(task_vector=synthetic_vector)
|
| 67 |
+
actions = [runtime.step(agent_img, wrist_img) for _ in range(16)]
|
| 68 |
+
print(f" -> Successfully stepped 16 cycles with zero task specification per step.")
|
| 69 |
+
print(f" -> Step 0: {np.round(actions[0], 4)}")
|
| 70 |
+
print(f" -> Step 8 (replanned): {np.round(actions[8], 4)}")
|
| 71 |
+
|
| 72 |
+
# 6. Benchmark High-Frequency Control Loop (Vector Already in GPU VRAM)
|
| 73 |
+
print("\n6. Benchmarking 100 Consecutive Chunks (Active Vector in GPU VRAM)...")
|
| 74 |
+
N = 100
|
| 75 |
+
t0 = time.perf_counter()
|
| 76 |
+
for _ in range(N):
|
| 77 |
+
_ = runtime.predict_chunk(agent_img, wrist_img)
|
| 78 |
+
mean_chunk_ms = (time.perf_counter() - t0) * 1000 / N
|
| 79 |
+
action_hz = 8.0 / (mean_chunk_ms / 1000.0)
|
| 80 |
+
|
| 81 |
+
print(f" -> Mean Chunk Latency: {mean_chunk_ms:.3f} ms / chunk")
|
| 82 |
+
print(f" -> Effective Action Bandwidth: {action_hz:.1f} Hz (>1,000 Hz target!)")
|
| 83 |
+
|
| 84 |
+
print("\n" + "=" * 75)
|
| 85 |
+
print(" ALL VECTOR-FIRST & PERTURBATION TESTS COMPLETED SUCCESSFULLY!")
|
| 86 |
+
print("=" * 75)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
if __name__ == "__main__":
|
| 90 |
+
test_vector_and_perturbations()
|
text_encoder.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Text conditioning encoder for Parvulus."""
|
| 2 |
+
|
| 3 |
+
from typing import Dict, Optional, Sequence
|
| 4 |
+
import torch
|
| 5 |
+
import torch.nn as nn
|
| 6 |
+
from transformers import AutoTokenizer, T5EncoderModel
|
| 7 |
+
|
| 8 |
+
DEFAULT_T5_MODEL = "google-t5/t5-small"
|
| 9 |
+
DEFAULT_T5_REVISION = "df1b051c49625cf57a3d0d8d3863ed4d13564fe4"
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ParvulusTextEncoder(nn.Module):
|
| 13 |
+
"""Mean-pools T5-small token embeddings and projects them to cond_dim."""
|
| 14 |
+
|
| 15 |
+
def __init__(
|
| 16 |
+
self,
|
| 17 |
+
cond_dim: int = 256,
|
| 18 |
+
model_name: str = DEFAULT_T5_MODEL,
|
| 19 |
+
revision: str = DEFAULT_T5_REVISION,
|
| 20 |
+
freeze_backbone: bool = True,
|
| 21 |
+
):
|
| 22 |
+
super().__init__()
|
| 23 |
+
self.cond_dim = cond_dim
|
| 24 |
+
self.model_name = model_name
|
| 25 |
+
self.revision = revision
|
| 26 |
+
self.freeze_backbone = freeze_backbone
|
| 27 |
+
|
| 28 |
+
self.tokenizer = AutoTokenizer.from_pretrained(model_name, revision=revision)
|
| 29 |
+
self.t5 = T5EncoderModel.from_pretrained(model_name, revision=revision)
|
| 30 |
+
if freeze_backbone:
|
| 31 |
+
self.t5.requires_grad_(False)
|
| 32 |
+
self.t5.eval()
|
| 33 |
+
|
| 34 |
+
self.proj = nn.Sequential(
|
| 35 |
+
nn.Linear(self.t5.config.d_model, cond_dim),
|
| 36 |
+
nn.SiLU(),
|
| 37 |
+
nn.Linear(cond_dim, cond_dim),
|
| 38 |
+
)
|
| 39 |
+
self._embed_cache: Dict[str, torch.Tensor] = {}
|
| 40 |
+
|
| 41 |
+
def train(self, mode: bool = True):
|
| 42 |
+
super().train(mode)
|
| 43 |
+
if self.freeze_backbone:
|
| 44 |
+
self.t5.eval()
|
| 45 |
+
return self
|
| 46 |
+
|
| 47 |
+
@staticmethod
|
| 48 |
+
def _mean_pool(last_hidden: torch.Tensor, attn_mask: torch.Tensor) -> torch.Tensor:
|
| 49 |
+
mask = attn_mask.unsqueeze(-1).to(last_hidden.dtype)
|
| 50 |
+
summed = (last_hidden * mask).sum(dim=1)
|
| 51 |
+
count = mask.sum(dim=1).clamp(min=1e-6)
|
| 52 |
+
return summed / count
|
| 53 |
+
|
| 54 |
+
def _t5_embed(self, texts: Sequence[str]) -> torch.Tensor:
|
| 55 |
+
device = next(self.t5.parameters()).device
|
| 56 |
+
tokens = self.tokenizer(
|
| 57 |
+
list(texts), padding=True, truncation=True, return_tensors="pt"
|
| 58 |
+
).to(device)
|
| 59 |
+
out = self.t5(**tokens).last_hidden_state
|
| 60 |
+
return self._mean_pool(out, tokens["attention_mask"])
|
| 61 |
+
|
| 62 |
+
def forward(self, texts: Sequence[str]) -> torch.Tensor:
|
| 63 |
+
device = next(self.parameters()).device
|
| 64 |
+
if self.freeze_backbone:
|
| 65 |
+
missing = [t for t in dict.fromkeys(texts) if t not in self._embed_cache]
|
| 66 |
+
if missing:
|
| 67 |
+
with torch.no_grad():
|
| 68 |
+
embeds = self._t5_embed(missing)
|
| 69 |
+
for t, e in zip(missing, embeds):
|
| 70 |
+
self._embed_cache[t] = e.detach()
|
| 71 |
+
pooled = torch.stack([self._embed_cache[t] for t in texts], dim=0).to(device)
|
| 72 |
+
else:
|
| 73 |
+
pooled = self._t5_embed(texts).to(device)
|
| 74 |
+
return self.proj(pooled)
|