Parvulus / parvulus_runtime.py
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
13.8 kB
"""Standalone high-performance policy runtime for Parvulus via native C ABI ctypes."""
import ctypes
import os
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Union
import numpy as np
_MODULE_DIR = Path(__file__).resolve().parent
_C_ENGINE_DIR = _MODULE_DIR / "c_engine"
_LIB_PATH = _C_ENGINE_DIR / "libparvulus.so"
_lib: Optional[ctypes.CDLL] = None
def _load_c_abi() -> ctypes.CDLL:
global _lib
if _lib is not None:
return _lib
if not _LIB_PATH.exists():
try:
import subprocess
print(f"[ParvulusRuntime] Building {_LIB_PATH.name}...")
subprocess.run(["make", "libparvulus.so"], cwd=str(_C_ENGINE_DIR), check=True)
except Exception:
pass
if not _LIB_PATH.exists():
raise FileNotFoundError(
f"Pure CUDA shared library not found at {_LIB_PATH}. "
f"Run 'make' or './build.sh' inside {_C_ENGINE_DIR} first."
)
lib = ctypes.CDLL(str(_LIB_PATH))
lib.parvulus_create.argtypes = [ctypes.c_int]
lib.parvulus_create.restype = ctypes.c_void_p
lib.parvulus_load.argtypes = [
ctypes.c_void_p,
ctypes.c_char_p,
ctypes.c_char_p,
ctypes.c_char_p,
]
lib.parvulus_load.restype = ctypes.c_bool
lib.parvulus_set_task_vector.argtypes = [
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_float),
]
lib.parvulus_set_task_vector.restype = ctypes.c_bool
lib.parvulus_set_task_string.argtypes = [
ctypes.c_void_p,
ctypes.c_char_p,
]
lib.parvulus_set_task_string.restype = ctypes.c_bool
lib.parvulus_predict_chunk.argtypes = [
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_uint8),
ctypes.POINTER(ctypes.c_uint8),
ctypes.c_char_p,
ctypes.POINTER(ctypes.c_float),
ctypes.c_int,
ctypes.POINTER(ctypes.c_float),
]
lib.parvulus_predict_chunk.restype = ctypes.c_bool
lib.parvulus_predict_chunk_with_vector.argtypes = [
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_uint8),
ctypes.POINTER(ctypes.c_uint8),
ctypes.POINTER(ctypes.c_float),
ctypes.POINTER(ctypes.c_float),
ctypes.c_int,
ctypes.POINTER(ctypes.c_float),
]
lib.parvulus_predict_chunk_with_vector.restype = ctypes.c_bool
lib.parvulus_register_task_embedding.argtypes = [
ctypes.c_void_p,
ctypes.c_char_p,
ctypes.POINTER(ctypes.c_float),
]
lib.parvulus_register_task_embedding.restype = ctypes.c_bool
lib.parvulus_destroy.argtypes = [ctypes.c_void_p]
lib.parvulus_destroy.restype = None
_lib = lib
return _lib
class ParvulusRuntime:
"""Standalone high-performance Parvulus policy inference engine (Vector-First)."""
def __init__(
self,
device_id: int = 0,
weights_path: Optional[Union[str, Path]] = None,
embeddings_path: Optional[Union[str, Path]] = None,
scale_path: Optional[Union[str, Path]] = None,
chunk_size: int = 8,
replan_stride: int = 8,
num_euler_steps: int = 10,
):
self.device_id = device_id
self.chunk_size = chunk_size
self.replan_stride = replan_stride
self.num_euler_steps = num_euler_steps
if weights_path is None:
weights_path = _C_ENGINE_DIR / "parvulus_model_weights.bin"
if embeddings_path is None:
embeddings_path = _C_ENGINE_DIR / "task_embeddings.bin"
if scale_path is None:
scale_path = _C_ENGINE_DIR / "action_scale.bin"
self.weights_path = Path(weights_path)
self.embeddings_path = Path(embeddings_path) if embeddings_path else None
self.scale_path = Path(scale_path) if scale_path else None
if not self.weights_path.exists():
print(f"[ParvulusRuntime] Generating {self.weights_path.name} from SafeTensors...")
import sys
if str(_C_ENGINE_DIR) not in sys.path:
sys.path.insert(0, str(_C_ENGINE_DIR))
from export_from_safetensors import export_binary_weights, export_action_scale, export_task_embeddings
st_path = _MODULE_DIR / "parvulus.safetensors"
if not st_path.exists():
st_path = _MODULE_DIR / "model.safetensors"
export_binary_weights(str(st_path), str(self.weights_path))
if self.scale_path and not self.scale_path.exists():
export_action_scale(str(_MODULE_DIR / "action_scale.json"), str(self.scale_path))
if self.embeddings_path and not self.embeddings_path.exists() and (_C_ENGINE_DIR / "canonical_tasks.json").exists():
export_task_embeddings(str(_C_ENGINE_DIR / "canonical_tasks.json"), str(st_path), str(self.embeddings_path))
self._c_lib = _load_c_abi()
self.handle = self._c_lib.parvulus_create(self.device_id)
if not self.handle:
raise RuntimeError(f"Failed to create Parvulus C/CUDA engine on GPU {self.device_id}")
emb_c = str(self.embeddings_path).encode("utf-8") if (self.embeddings_path and self.embeddings_path.exists()) else None
scl_c = str(self.scale_path).encode("utf-8") if (self.scale_path and self.scale_path.exists()) else None
ok = self._c_lib.parvulus_load(
self.handle,
str(self.weights_path).encode("utf-8"),
emb_c,
scl_c,
)
if not ok:
self._c_lib.parvulus_destroy(self.handle)
self.handle = None
raise RuntimeError("Failed to load Parvulus model weights into GPU VRAM!")
self.current_task_desc: Optional[str] = None
self.current_task_vector: Optional[np.ndarray] = None
self.action_queue: List[np.ndarray] = []
self.stride_counter: int = 0
self._actions_buffer = np.zeros(56, dtype=np.float32)
self._onnx_session = None
self._tokenizer = None
self._torch_text_encoder = None
def __del__(self):
if hasattr(self, "handle") and self.handle is not None and self._c_lib is not None:
self._c_lib.parvulus_destroy(self.handle)
self.handle = None
def set_task_vector(self, task_vector: Union[np.ndarray, Sequence[float]]) -> bool:
"""Sets the active 256-dim task conditioning vector on the GPU."""
vec_c = np.ascontiguousarray(task_vector, dtype=np.float32).flatten()[:256]
vec_ptr = vec_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
ok = bool(self._c_lib.parvulus_set_task_vector(self.handle, vec_ptr))
if ok:
self.current_task_vector = vec_c
return ok
def reset(
self,
task_description: Optional[str] = None,
task_vector: Optional[Union[np.ndarray, Sequence[float]]] = None,
):
"""Resets action queue and updates task conditioning for a new episode.
Accepts:
- task_vector: 256-dim float vector (first-class input).
- task_description: text instruction string (automatically handles perturbations
by encoding them on the fly into 256-dim conditioning vectors).
"""
if task_vector is not None:
self.set_task_vector(task_vector)
self.current_task_desc = task_description or "<custom_vector>"
elif task_description is not None:
self.current_task_desc = task_description
task_bytes = task_description.encode("utf-8")
if not self._c_lib.parvulus_set_task_string(self.handle, task_bytes):
encoded_vec = self.encode_text(task_description)
self.set_task_vector(encoded_vec)
self.action_queue.clear()
self.stride_counter = 0
def register_task_embedding(self, task_description: str, text_embedding: np.ndarray) -> bool:
"""Registers a 256-dim embedding vector for a task description in engine memory."""
task_bytes = task_description.encode("utf-8")
emb_c = np.ascontiguousarray(text_embedding, dtype=np.float32).flatten()[:256]
emb_ptr = emb_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
return bool(self._c_lib.parvulus_register_task_embedding(self.handle, task_bytes, emb_ptr))
def encode_text(self, text: str) -> np.ndarray:
"""Encodes an arbitrary natural language prompt into a 256-dim vector."""
onnx_path = _C_ENGINE_DIR / "parvulus_text_encoder.onnx"
if not onnx_path.exists() and os.environ.get("PARVULUS_HF_REPO"):
try:
from huggingface_hub import hf_hub_download
hf_hub_download(
repo_id=os.environ["PARVULUS_HF_REPO"],
filename="parvulus_text_encoder.onnx",
subfolder="c_engine",
local_dir=str(_MODULE_DIR),
)
except Exception:
pass
if onnx_path.exists():
try:
import onnxruntime as ort
from transformers import AutoTokenizer
if self._onnx_session is None:
available = ort.get_available_providers()
providers = [p for p in ["CUDAExecutionProvider", "CPUExecutionProvider"] if p in available]
self._onnx_session = ort.InferenceSession(str(onnx_path), providers=providers)
self._tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small")
tokens = self._tokenizer([text], padding=True, truncation=True, return_tensors="np")
ort_out = self._onnx_session.run(
None,
{
"input_ids": tokens["input_ids"].astype(np.int64),
"attention_mask": tokens["attention_mask"].astype(np.int64),
},
)[0]
return ort_out[0].astype(np.float32)
except Exception:
pass
try:
import torch
from text_encoder import ParvulusTextEncoder
from safetensors.torch import load_file
if self._torch_text_encoder is None:
te = ParvulusTextEncoder().cpu().eval()
st_path = _MODULE_DIR / "parvulus.safetensors"
if st_path.exists():
weights = load_file(str(st_path))
proj_dict = {
"0.weight": weights["text_encoder.proj.0.weight"],
"0.bias": weights["text_encoder.proj.0.bias"],
"2.weight": weights["text_encoder.proj.2.weight"],
"2.bias": weights["text_encoder.proj.2.bias"],
}
te.proj.load_state_dict(proj_dict)
self._torch_text_encoder = te
with torch.no_grad():
emb = self._torch_text_encoder([text]).cpu().numpy()[0]
return emb.astype(np.float32)
except Exception as e:
raise RuntimeError(
f"Failed to dynamically encode text prompt '{text}': {e}. "
"Ensure onnxruntime or transformers/torch is installed."
)
def predict_chunk(
self,
agentview_img: np.ndarray,
wrist_img: np.ndarray,
task_vector: Optional[Union[np.ndarray, Sequence[float]]] = None,
task_description: Optional[str] = None,
custom_z0: Optional[np.ndarray] = None,
) -> np.ndarray:
"""Predicts an [8, 7] action chunk using active or newly provided task conditioning."""
if task_vector is not None or task_description is not None:
self.reset(task_description=task_description, task_vector=task_vector)
agent_c = np.ascontiguousarray(agentview_img, dtype=np.uint8)
wrist_c = np.ascontiguousarray(wrist_img, dtype=np.uint8)
agent_ptr = agent_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
wrist_ptr = wrist_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8))
out_ptr = self._actions_buffer.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
z0_ptr = None
if custom_z0 is not None:
z0_c = np.ascontiguousarray(custom_z0, dtype=np.float32)
z0_ptr = z0_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
ok = self._c_lib.parvulus_predict_chunk(
self.handle,
agent_ptr,
wrist_ptr,
None,
out_ptr,
self.num_euler_steps,
z0_ptr,
)
if not ok:
raise RuntimeError("Pure CUDA predict_chunk failed! Check task conditioning vector.")
return self._actions_buffer.reshape(8, 7).copy()
def predict_chunk_with_emb(
self,
agentview_img: np.ndarray,
wrist_img: np.ndarray,
text_embedding: np.ndarray,
custom_z0: Optional[np.ndarray] = None,
) -> np.ndarray:
"""Alias for predict_chunk(agentview_img, wrist_img, task_vector=text_embedding)."""
return self.predict_chunk(
agentview_img=agentview_img,
wrist_img=wrist_img,
task_vector=text_embedding,
custom_z0=custom_z0,
)
def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray:
"""Executes a single 7-DoF step, automatically managing the receding horizon action queue."""
if not self.action_queue or self.stride_counter >= self.replan_stride:
chunk = self.predict_chunk(agentview_img, wrist_img)
self.action_queue = list(chunk[: self.replan_stride])
self.stride_counter = 0
action = self.action_queue.pop(0)
self.stride_counter += 1
return action