"""Inference client and high-level runtime for Parvulus robot policies.""" from pathlib import Path from typing import Dict, List, Optional, Sequence, Tuple, Union import cv2 import numpy as np import torch from PIL import Image from action_utils import DEFAULT_SCALE_PATH, denormalize_action, load_action_scale from parvulus import ParvulusModel class ParvulusInference: """High-level inference runner for Parvulus robot policies.""" def __init__( self, model_path: Union[str, Path] = ".", scale_path: Union[str, Path] = DEFAULT_SCALE_PATH, device: str = "auto", use_kv_cache: bool = True, load_auxiliary: bool = False, chunk_size: int = 8, replan_stride: int = 8, num_ode_steps: int = 10, resolution: int = 256, flip_convention: bool = True, gamma: Optional[float] = None, auto_gamma: bool = False, backend: str = "pytorch", ): """Initializes Parvulus inference runner. Args: model_path: Directory containing parvulus.safetensors and config.json. scale_path: Path to action_scale.json. device: 'cuda', 'cpu', 'xpu', 'hybrid', or 'auto'. 'hybrid' executes visual perception on Intel iGPU (xpu) and the ODE loop on CPU. use_kv_cache: Precomputes visual keys/values for fast 38ms action chunking. load_auxiliary: If True, also loads video, change, and self-reconstruction heads. chunk_size: Action chunk horizon (default 8). replan_stride: Frequency to replan in step() (default 8). num_ode_steps: Euler integration steps (default 10). resolution: Camera resolution (256x256). flip_convention: Applies standard Robosuite/LIBERO camera orientation flip. gamma: Explicit exposure correction gamma. auto_gamma: Adaptively brightens underexposed manipulation scenes. backend: Runtime execution engine: 'pytorch' (standard), 'cuda_c' (pure C ABI), or 'cuda_graph' (CUDA Graph replay, 7.68ms / chunk). """ self.backend = backend self.chunk_size = chunk_size self.replan_stride = replan_stride self.num_ode_steps = num_ode_steps self.resolution = resolution self.flip_convention = flip_convention self.use_kv_cache = use_kv_cache self.gamma = gamma self.auto_gamma = auto_gamma if backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"): from c_engine.infer_c import ParvulusEngine, ParvulusCUDAGraphEngine dev_id = 0 if isinstance(device, str) and "cuda:" in device: dev_id = int(device.split(":")[-1]) elif isinstance(device, torch.device) and device.index is not None: dev_id = device.index if backend in ("parvulus", "pure_cuda", "cuda_c"): self._c_engine = ParvulusEngine( device_id=dev_id, chunk_size=chunk_size, replan_stride=replan_stride, num_ode_steps=num_ode_steps, ) elif backend == "cuda_graph": self._c_engine = ParvulusCUDAGraphEngine( device_id=dev_id, chunk_size=chunk_size, replan_stride=replan_stride, num_ode_steps=num_ode_steps, ) return self.is_hybrid = False if device == "auto": if torch.cuda.is_available(): self.device = torch.device("cuda") elif hasattr(torch, "xpu") and torch.xpu.is_available(): self.device = torch.device("xpu") else: self.device = torch.device("cpu") elif device == "hybrid": if not (hasattr(torch, "xpu") and torch.xpu.is_available()): print("[Warning] Device 'hybrid' requested but torch.xpu is unavailable. Falling back to CPU.") self.device = torch.device("cpu") else: self.is_hybrid = True self.device = torch.device("cpu") self.xpu_device = torch.device("xpu") else: self.device = torch.device(device) eff_gamma = self.gamma if (self.gamma is not None and self.gamma != 1.0) else 0.55 self._gamma_lut = np.array([((i / 255.0) ** eff_gamma) * 255.0 for i in range(256)]).astype(np.uint8) self.action_min, self.action_max = load_action_scale(scale_path) self.action_min = self.action_min.to(self.device) self.action_max = self.action_max.to(self.device) self.model = ParvulusModel.from_pretrained( pretrained_path=model_path, load_auxiliary=load_auxiliary, load_text_encoder=True, device=self.device, ) if self.is_hybrid: self.model.encoder = self.model.encoder.to(self.xpu_device).eval() self.current_task_desc: Optional[str] = None self.current_text_emb: Optional[torch.Tensor] = None self.current_text_emb_xpu: Optional[torch.Tensor] = None self.action_queue: List[np.ndarray] = [] self.stride_counter: int = 0 @classmethod def from_pretrained(cls, pretrained_path: Union[str, Path] = ".", **kwargs) -> "ParvulusInference": return cls(model_path=pretrained_path, **kwargs) def preprocess_image(self, img: np.ndarray, target_device: Optional[torch.device] = None) -> torch.Tensor: """Preprocesses [H, W, 3] uint8 image -> [1, 3, res, res] tensor in [-1, 1].""" dev = target_device or self.device if self.flip_convention: img = np.ascontiguousarray(img[::-1, ::-1]) if img.shape[0] != self.resolution or img.shape[1] != self.resolution: img = np.array(Image.fromarray(img).resize((self.resolution, self.resolution), Image.BILINEAR)) if self.gamma is not None and self.gamma != 1.0: img = cv2.LUT(img, self._gamma_lut) elif self.auto_gamma: mean_y = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY).mean() if mean_y < 112.0: img = cv2.LUT(img, self._gamma_lut) t = torch.from_numpy(img).float() / 255.0 t = t.permute(2, 0, 1) * 2.0 - 1.0 return t.unsqueeze(0).to(dev) def reset(self, task_description: Optional[str] = None, text_emb: Optional[torch.Tensor] = None): """Resets policy state for a new episode.""" if self.backend in ("parvulus", "pure_cuda", "cuda_c"): if text_emb is not None: vec = text_emb.detach().cpu().numpy() self._c_engine.reset(task_description=task_description, task_vector=vec) elif task_description is not None: # If model with text_encoder is loaded, encode on the fly to support perturbations if hasattr(self, "model") and hasattr(self.model, "text_encoder") and self.model.text_encoder is not None: with torch.no_grad(): vec = self.model.text_encoder([task_description]).detach().cpu().numpy()[0] self._c_engine.reset(task_description=task_description, task_vector=vec) else: self._c_engine.reset(task_description=task_description) else: self._c_engine.reset() return elif self.backend == "cuda_graph": self._c_engine.reset(task_description) return self.current_task_desc = task_description if text_emb is not None: self.current_text_emb = text_emb if text_emb.dim() == 2 else text_emb.unsqueeze(0).to(self.device) elif task_description is not None: with torch.no_grad(): self.current_text_emb = self.model.text_encoder([task_description]).to(self.device) else: self.current_text_emb = None if self.is_hybrid and self.current_text_emb is not None: self.current_text_emb_xpu = self.current_text_emb.to(self.xpu_device) self.action_queue.clear() self.stride_counter = 0 @torch.no_grad() def predict_chunk( self, agentview_img: np.ndarray, wrist_img: np.ndarray, task_description: Optional[str] = None, denormalize: bool = True, ) -> np.ndarray: """Predicts full 8-step continuous action chunk [8, 7].""" if self.backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"): return self._c_engine.predict_chunk(agentview_img, wrist_img, task_description=task_description) if task_description is not None: self.reset(task_description) elif self.current_text_emb is None: raise ValueError("Call reset(task_description=...) before predicting actions!") if self.is_hybrid: agent_t = self.preprocess_image(agentview_img, target_device=self.xpu_device) wrist_t = self.preprocess_image(wrist_img, target_device=self.xpu_device) obs_xpu = torch.cat([agent_t, wrist_t], dim=1) _, z_vis_xpu = self.model.encoder(obs_xpu, self.current_text_emb_xpu) torch.xpu.synchronize() z_vis = z_vis_xpu.to(self.device) z = torch.randn(1, self.chunk_size, self.model.action_dim, device=self.device) kv_layers = self.model._get_kv_layers() kv_caches = [layer.precompute_kv(z_vis) for layer in kv_layers] dt = 1.0 / self.num_ode_steps for step in range(self.num_ode_steps): sigma = torch.tensor([step * dt], device=self.device) cond = self.current_text_emb + self.model.action_t_mlp(sigma) h = self.model.action_decoder.action_in_proj(z) + self.model.action_decoder.pos_emb[:, :self.chunk_size, :] for layer, (k_v, v_v) in zip(kv_layers, kv_caches): h = layer.forward_with_kv(h, k_v, v_v, cond) v_pred = self.model.action_decoder.out_head(self.model.action_decoder.final_norm(h)) z = z + v_pred * dt chunk_norm = z.clamp(-1.0, 1.0) else: agent_t = self.preprocess_image(agentview_img) wrist_t = self.preprocess_image(wrist_img) obs = torch.cat([agent_t, wrist_t], dim=1) chunk_norm = self.model.predict_action_chunk( obs, text_emb=self.current_text_emb, num_ode_steps=self.num_ode_steps, use_kv_cache=self.use_kv_cache, ) if denormalize: chunk = denormalize_action(chunk_norm.squeeze(0), self.action_min, self.action_max) else: chunk = chunk_norm.squeeze(0) return chunk.cpu().numpy() def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray: """Returns single 7-DoF action, replanning every C=replan_stride steps.""" if self.backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"): return self._c_engine.step(agentview_img, wrist_img) if not self.action_queue or self.stride_counter >= self.replan_stride: chunk = self.predict_chunk(agentview_img, wrist_img, denormalize=True) self.action_queue = list(chunk[: self.replan_stride]) self.stride_counter = 0 action = self.action_queue.pop(0) self.stride_counter += 1 return action @torch.no_grad() def rollout_step( self, agentview_img: np.ndarray, wrist_img: np.ndarray, task_description: Optional[str] = None, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: """Auxiliary world-model step: predicts action chunk AND next dual-camera observation. Returns: action_chunk: [8, 7] float32 action chunk. next_agent_img: [256, 256, 3] uint8 predicted agentview. next_wrist_img: [256, 256, 3] uint8 predicted wristview. """ action_chunk = self.predict_chunk(agentview_img, wrist_img, task_description=task_description) agent_t = self.preprocess_image(agentview_img) wrist_t = self.preprocess_image(wrist_img) obs = torch.cat([agent_t, wrist_t], dim=1) pred_obs = self.model.predict_next_frame(obs, text_emb=self.current_text_emb, num_ode_steps=self.num_ode_steps) pred_obs = ((pred_obs.squeeze(0) + 1.0) / 2.0 * 255.0).clamp(0, 255).byte().permute(1, 2, 0).cpu().numpy() next_agent = pred_obs[:, :, 0:3] next_wrist = pred_obs[:, :, 3:6] if self.flip_convention: next_agent = np.ascontiguousarray(next_agent[::-1, ::-1]) next_wrist = np.ascontiguousarray(next_wrist[::-1, ::-1]) return action_chunk, next_agent, next_wrist ParvulusInfer = ParvulusInference __all__ = ["ParvulusInference", "ParvulusInfer"]