Download parvulus_runtime.py from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 13.8 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/parvulus_runtime.py
- Command line
-
hf download hf://aasivas/Parvulus/parvulus_runtime.py
-
curl -L -o parvulus_runtime.py https://huggingface.co/aasivas/Parvulus/resolve/main/parvulus_runtime.py
13.8 kB
| """Standalone high-performance policy runtime for Parvulus via native C ABI ctypes.""" | |
| import ctypes | |
| import os | |
| from pathlib import Path | |
| from typing import Dict, List, Optional, Sequence, Union | |
| import numpy as np | |
| _MODULE_DIR = Path(__file__).resolve().parent | |
| _C_ENGINE_DIR = _MODULE_DIR / "c_engine" | |
| _LIB_PATH = _C_ENGINE_DIR / "libparvulus.so" | |
| _lib: Optional[ctypes.CDLL] = None | |
| def _load_c_abi() -> ctypes.CDLL: | |
| global _lib | |
| if _lib is not None: | |
| return _lib | |
| if not _LIB_PATH.exists(): | |
| try: | |
| import subprocess | |
| print(f"[ParvulusRuntime] Building {_LIB_PATH.name}...") | |
| subprocess.run(["make", "libparvulus.so"], cwd=str(_C_ENGINE_DIR), check=True) | |
| except Exception: | |
| pass | |
| if not _LIB_PATH.exists(): | |
| raise FileNotFoundError( | |
| f"Pure CUDA shared library not found at {_LIB_PATH}. " | |
| f"Run 'make' or './build.sh' inside {_C_ENGINE_DIR} first." | |
| ) | |
| lib = ctypes.CDLL(str(_LIB_PATH)) | |
| lib.parvulus_create.argtypes = [ctypes.c_int] | |
| lib.parvulus_create.restype = ctypes.c_void_p | |
| lib.parvulus_load.argtypes = [ | |
| ctypes.c_void_p, | |
| ctypes.c_char_p, | |
| ctypes.c_char_p, | |
| ctypes.c_char_p, | |
| ] | |
| lib.parvulus_load.restype = ctypes.c_bool | |
| lib.parvulus_set_task_vector.argtypes = [ | |
| ctypes.c_void_p, | |
| ctypes.POINTER(ctypes.c_float), | |
| ] | |
| lib.parvulus_set_task_vector.restype = ctypes.c_bool | |
| lib.parvulus_set_task_string.argtypes = [ | |
| ctypes.c_void_p, | |
| ctypes.c_char_p, | |
| ] | |
| lib.parvulus_set_task_string.restype = ctypes.c_bool | |
| lib.parvulus_predict_chunk.argtypes = [ | |
| ctypes.c_void_p, | |
| ctypes.POINTER(ctypes.c_uint8), | |
| ctypes.POINTER(ctypes.c_uint8), | |
| ctypes.c_char_p, | |
| ctypes.POINTER(ctypes.c_float), | |
| ctypes.c_int, | |
| ctypes.POINTER(ctypes.c_float), | |
| ] | |
| lib.parvulus_predict_chunk.restype = ctypes.c_bool | |
| lib.parvulus_predict_chunk_with_vector.argtypes = [ | |
| ctypes.c_void_p, | |
| ctypes.POINTER(ctypes.c_uint8), | |
| ctypes.POINTER(ctypes.c_uint8), | |
| ctypes.POINTER(ctypes.c_float), | |
| ctypes.POINTER(ctypes.c_float), | |
| ctypes.c_int, | |
| ctypes.POINTER(ctypes.c_float), | |
| ] | |
| lib.parvulus_predict_chunk_with_vector.restype = ctypes.c_bool | |
| lib.parvulus_register_task_embedding.argtypes = [ | |
| ctypes.c_void_p, | |
| ctypes.c_char_p, | |
| ctypes.POINTER(ctypes.c_float), | |
| ] | |
| lib.parvulus_register_task_embedding.restype = ctypes.c_bool | |
| lib.parvulus_destroy.argtypes = [ctypes.c_void_p] | |
| lib.parvulus_destroy.restype = None | |
| _lib = lib | |
| return _lib | |
| class ParvulusRuntime: | |
| """Standalone high-performance Parvulus policy inference engine (Vector-First).""" | |
| def __init__( | |
| self, | |
| device_id: int = 0, | |
| weights_path: Optional[Union[str, Path]] = None, | |
| embeddings_path: Optional[Union[str, Path]] = None, | |
| scale_path: Optional[Union[str, Path]] = None, | |
| chunk_size: int = 8, | |
| replan_stride: int = 8, | |
| num_euler_steps: int = 10, | |
| ): | |
| self.device_id = device_id | |
| self.chunk_size = chunk_size | |
| self.replan_stride = replan_stride | |
| self.num_euler_steps = num_euler_steps | |
| if weights_path is None: | |
| weights_path = _C_ENGINE_DIR / "parvulus_model_weights.bin" | |
| if embeddings_path is None: | |
| embeddings_path = _C_ENGINE_DIR / "task_embeddings.bin" | |
| if scale_path is None: | |
| scale_path = _C_ENGINE_DIR / "action_scale.bin" | |
| self.weights_path = Path(weights_path) | |
| self.embeddings_path = Path(embeddings_path) if embeddings_path else None | |
| self.scale_path = Path(scale_path) if scale_path else None | |
| if not self.weights_path.exists(): | |
| print(f"[ParvulusRuntime] Generating {self.weights_path.name} from SafeTensors...") | |
| import sys | |
| if str(_C_ENGINE_DIR) not in sys.path: | |
| sys.path.insert(0, str(_C_ENGINE_DIR)) | |
| from export_from_safetensors import export_binary_weights, export_action_scale, export_task_embeddings | |
| st_path = _MODULE_DIR / "parvulus.safetensors" | |
| if not st_path.exists(): | |
| st_path = _MODULE_DIR / "model.safetensors" | |
| export_binary_weights(str(st_path), str(self.weights_path)) | |
| if self.scale_path and not self.scale_path.exists(): | |
| export_action_scale(str(_MODULE_DIR / "action_scale.json"), str(self.scale_path)) | |
| if self.embeddings_path and not self.embeddings_path.exists() and (_C_ENGINE_DIR / "canonical_tasks.json").exists(): | |
| export_task_embeddings(str(_C_ENGINE_DIR / "canonical_tasks.json"), str(st_path), str(self.embeddings_path)) | |
| self._c_lib = _load_c_abi() | |
| self.handle = self._c_lib.parvulus_create(self.device_id) | |
| if not self.handle: | |
| raise RuntimeError(f"Failed to create Parvulus C/CUDA engine on GPU {self.device_id}") | |
| emb_c = str(self.embeddings_path).encode("utf-8") if (self.embeddings_path and self.embeddings_path.exists()) else None | |
| scl_c = str(self.scale_path).encode("utf-8") if (self.scale_path and self.scale_path.exists()) else None | |
| ok = self._c_lib.parvulus_load( | |
| self.handle, | |
| str(self.weights_path).encode("utf-8"), | |
| emb_c, | |
| scl_c, | |
| ) | |
| if not ok: | |
| self._c_lib.parvulus_destroy(self.handle) | |
| self.handle = None | |
| raise RuntimeError("Failed to load Parvulus model weights into GPU VRAM!") | |
| self.current_task_desc: Optional[str] = None | |
| self.current_task_vector: Optional[np.ndarray] = None | |
| self.action_queue: List[np.ndarray] = [] | |
| self.stride_counter: int = 0 | |
| self._actions_buffer = np.zeros(56, dtype=np.float32) | |
| self._onnx_session = None | |
| self._tokenizer = None | |
| self._torch_text_encoder = None | |
| def __del__(self): | |
| if hasattr(self, "handle") and self.handle is not None and self._c_lib is not None: | |
| self._c_lib.parvulus_destroy(self.handle) | |
| self.handle = None | |
| def set_task_vector(self, task_vector: Union[np.ndarray, Sequence[float]]) -> bool: | |
| """Sets the active 256-dim task conditioning vector on the GPU.""" | |
| vec_c = np.ascontiguousarray(task_vector, dtype=np.float32).flatten()[:256] | |
| vec_ptr = vec_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) | |
| ok = bool(self._c_lib.parvulus_set_task_vector(self.handle, vec_ptr)) | |
| if ok: | |
| self.current_task_vector = vec_c | |
| return ok | |
| def reset( | |
| self, | |
| task_description: Optional[str] = None, | |
| task_vector: Optional[Union[np.ndarray, Sequence[float]]] = None, | |
| ): | |
| """Resets action queue and updates task conditioning for a new episode. | |
| Accepts: | |
| - task_vector: 256-dim float vector (first-class input). | |
| - task_description: text instruction string (automatically handles perturbations | |
| by encoding them on the fly into 256-dim conditioning vectors). | |
| """ | |
| if task_vector is not None: | |
| self.set_task_vector(task_vector) | |
| self.current_task_desc = task_description or "<custom_vector>" | |
| elif task_description is not None: | |
| self.current_task_desc = task_description | |
| task_bytes = task_description.encode("utf-8") | |
| if not self._c_lib.parvulus_set_task_string(self.handle, task_bytes): | |
| encoded_vec = self.encode_text(task_description) | |
| self.set_task_vector(encoded_vec) | |
| self.action_queue.clear() | |
| self.stride_counter = 0 | |
| def register_task_embedding(self, task_description: str, text_embedding: np.ndarray) -> bool: | |
| """Registers a 256-dim embedding vector for a task description in engine memory.""" | |
| task_bytes = task_description.encode("utf-8") | |
| emb_c = np.ascontiguousarray(text_embedding, dtype=np.float32).flatten()[:256] | |
| emb_ptr = emb_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) | |
| return bool(self._c_lib.parvulus_register_task_embedding(self.handle, task_bytes, emb_ptr)) | |
| def encode_text(self, text: str) -> np.ndarray: | |
| """Encodes an arbitrary natural language prompt into a 256-dim vector.""" | |
| onnx_path = _C_ENGINE_DIR / "parvulus_text_encoder.onnx" | |
| if not onnx_path.exists() and os.environ.get("PARVULUS_HF_REPO"): | |
| try: | |
| from huggingface_hub import hf_hub_download | |
| hf_hub_download( | |
| repo_id=os.environ["PARVULUS_HF_REPO"], | |
| filename="parvulus_text_encoder.onnx", | |
| subfolder="c_engine", | |
| local_dir=str(_MODULE_DIR), | |
| ) | |
| except Exception: | |
| pass | |
| if onnx_path.exists(): | |
| try: | |
| import onnxruntime as ort | |
| from transformers import AutoTokenizer | |
| if self._onnx_session is None: | |
| available = ort.get_available_providers() | |
| providers = [p for p in ["CUDAExecutionProvider", "CPUExecutionProvider"] if p in available] | |
| self._onnx_session = ort.InferenceSession(str(onnx_path), providers=providers) | |
| self._tokenizer = AutoTokenizer.from_pretrained("google-t5/t5-small") | |
| tokens = self._tokenizer([text], padding=True, truncation=True, return_tensors="np") | |
| ort_out = self._onnx_session.run( | |
| None, | |
| { | |
| "input_ids": tokens["input_ids"].astype(np.int64), | |
| "attention_mask": tokens["attention_mask"].astype(np.int64), | |
| }, | |
| )[0] | |
| return ort_out[0].astype(np.float32) | |
| except Exception: | |
| pass | |
| try: | |
| import torch | |
| from text_encoder import ParvulusTextEncoder | |
| from safetensors.torch import load_file | |
| if self._torch_text_encoder is None: | |
| te = ParvulusTextEncoder().cpu().eval() | |
| st_path = _MODULE_DIR / "parvulus.safetensors" | |
| if st_path.exists(): | |
| weights = load_file(str(st_path)) | |
| proj_dict = { | |
| "0.weight": weights["text_encoder.proj.0.weight"], | |
| "0.bias": weights["text_encoder.proj.0.bias"], | |
| "2.weight": weights["text_encoder.proj.2.weight"], | |
| "2.bias": weights["text_encoder.proj.2.bias"], | |
| } | |
| te.proj.load_state_dict(proj_dict) | |
| self._torch_text_encoder = te | |
| with torch.no_grad(): | |
| emb = self._torch_text_encoder([text]).cpu().numpy()[0] | |
| return emb.astype(np.float32) | |
| except Exception as e: | |
| raise RuntimeError( | |
| f"Failed to dynamically encode text prompt '{text}': {e}. " | |
| "Ensure onnxruntime or transformers/torch is installed." | |
| ) | |
| def predict_chunk( | |
| self, | |
| agentview_img: np.ndarray, | |
| wrist_img: np.ndarray, | |
| task_vector: Optional[Union[np.ndarray, Sequence[float]]] = None, | |
| task_description: Optional[str] = None, | |
| custom_z0: Optional[np.ndarray] = None, | |
| ) -> np.ndarray: | |
| """Predicts an [8, 7] action chunk using active or newly provided task conditioning.""" | |
| if task_vector is not None or task_description is not None: | |
| self.reset(task_description=task_description, task_vector=task_vector) | |
| agent_c = np.ascontiguousarray(agentview_img, dtype=np.uint8) | |
| wrist_c = np.ascontiguousarray(wrist_img, dtype=np.uint8) | |
| agent_ptr = agent_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8)) | |
| wrist_ptr = wrist_c.ctypes.data_as(ctypes.POINTER(ctypes.c_uint8)) | |
| out_ptr = self._actions_buffer.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) | |
| z0_ptr = None | |
| if custom_z0 is not None: | |
| z0_c = np.ascontiguousarray(custom_z0, dtype=np.float32) | |
| z0_ptr = z0_c.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) | |
| ok = self._c_lib.parvulus_predict_chunk( | |
| self.handle, | |
| agent_ptr, | |
| wrist_ptr, | |
| None, | |
| out_ptr, | |
| self.num_euler_steps, | |
| z0_ptr, | |
| ) | |
| if not ok: | |
| raise RuntimeError("Pure CUDA predict_chunk failed! Check task conditioning vector.") | |
| return self._actions_buffer.reshape(8, 7).copy() | |
| def predict_chunk_with_emb( | |
| self, | |
| agentview_img: np.ndarray, | |
| wrist_img: np.ndarray, | |
| text_embedding: np.ndarray, | |
| custom_z0: Optional[np.ndarray] = None, | |
| ) -> np.ndarray: | |
| """Alias for predict_chunk(agentview_img, wrist_img, task_vector=text_embedding).""" | |
| return self.predict_chunk( | |
| agentview_img=agentview_img, | |
| wrist_img=wrist_img, | |
| task_vector=text_embedding, | |
| custom_z0=custom_z0, | |
| ) | |
| def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray: | |
| """Executes a single 7-DoF step, automatically managing the receding horizon action queue.""" | |
| if not self.action_queue or self.stride_counter >= self.replan_stride: | |
| chunk = self.predict_chunk(agentview_img, wrist_img) | |
| self.action_queue = list(chunk[: self.replan_stride]) | |
| self.stride_counter = 0 | |
| action = self.action_queue.pop(0) | |
| self.stride_counter += 1 | |
| return action | |