Parvulus / infer.py
aasivas's picture
Upload folder using huggingface_hub
67c4f10 verified
Raw History Blame Contribute Delete
13.1 kB
"""Inference client and high-level runtime for Parvulus robot policies."""
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Tuple, Union
import cv2
import numpy as np
import torch
from PIL import Image
from action_utils import DEFAULT_SCALE_PATH, denormalize_action, load_action_scale
from parvulus import ParvulusModel
class ParvulusInference:
"""High-level inference runner for Parvulus robot policies."""
def __init__(
self,
model_path: Union[str, Path] = ".",
scale_path: Union[str, Path] = DEFAULT_SCALE_PATH,
device: str = "auto",
use_kv_cache: bool = True,
load_auxiliary: bool = False,
chunk_size: int = 8,
replan_stride: int = 8,
num_ode_steps: int = 10,
resolution: int = 256,
flip_convention: bool = True,
gamma: Optional[float] = None,
auto_gamma: bool = False,
backend: str = "pytorch",
):
"""Initializes Parvulus inference runner.
Args:
model_path: Directory containing parvulus.safetensors and config.json.
scale_path: Path to action_scale.json.
device: 'cuda', 'cpu', 'xpu', 'hybrid', or 'auto'.
'hybrid' executes visual perception on Intel iGPU (xpu) and the ODE loop on CPU.
use_kv_cache: Precomputes visual keys/values for fast 38ms action chunking.
load_auxiliary: If True, also loads video, change, and self-reconstruction heads.
chunk_size: Action chunk horizon (default 8).
replan_stride: Frequency to replan in step() (default 8).
num_ode_steps: Euler integration steps (default 10).
resolution: Camera resolution (256x256).
flip_convention: Applies standard Robosuite/LIBERO camera orientation flip.
gamma: Explicit exposure correction gamma.
auto_gamma: Adaptively brightens underexposed manipulation scenes.
backend: Runtime execution engine: 'pytorch' (standard), 'cuda_c' (pure C ABI),
or 'cuda_graph' (CUDA Graph replay, 7.68ms / chunk).
"""
self.backend = backend
self.chunk_size = chunk_size
self.replan_stride = replan_stride
self.num_ode_steps = num_ode_steps
self.resolution = resolution
self.flip_convention = flip_convention
self.use_kv_cache = use_kv_cache
self.gamma = gamma
self.auto_gamma = auto_gamma
if backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"):
from c_engine.infer_c import ParvulusEngine, ParvulusCUDAGraphEngine
dev_id = 0
if isinstance(device, str) and "cuda:" in device:
dev_id = int(device.split(":")[-1])
elif isinstance(device, torch.device) and device.index is not None:
dev_id = device.index
if backend in ("parvulus", "pure_cuda", "cuda_c"):
self._c_engine = ParvulusEngine(
device_id=dev_id,
chunk_size=chunk_size,
replan_stride=replan_stride,
num_ode_steps=num_ode_steps,
)
elif backend == "cuda_graph":
self._c_engine = ParvulusCUDAGraphEngine(
device_id=dev_id,
chunk_size=chunk_size,
replan_stride=replan_stride,
num_ode_steps=num_ode_steps,
)
return
self.is_hybrid = False
if device == "auto":
if torch.cuda.is_available():
self.device = torch.device("cuda")
elif hasattr(torch, "xpu") and torch.xpu.is_available():
self.device = torch.device("xpu")
else:
self.device = torch.device("cpu")
elif device == "hybrid":
if not (hasattr(torch, "xpu") and torch.xpu.is_available()):
print("[Warning] Device 'hybrid' requested but torch.xpu is unavailable. Falling back to CPU.")
self.device = torch.device("cpu")
else:
self.is_hybrid = True
self.device = torch.device("cpu")
self.xpu_device = torch.device("xpu")
else:
self.device = torch.device(device)
eff_gamma = self.gamma if (self.gamma is not None and self.gamma != 1.0) else 0.55
self._gamma_lut = np.array([((i / 255.0) ** eff_gamma) * 255.0 for i in range(256)]).astype(np.uint8)
self.action_min, self.action_max = load_action_scale(scale_path)
self.action_min = self.action_min.to(self.device)
self.action_max = self.action_max.to(self.device)
self.model = ParvulusModel.from_pretrained(
pretrained_path=model_path,
load_auxiliary=load_auxiliary,
load_text_encoder=True,
device=self.device,
)
if self.is_hybrid:
self.model.encoder = self.model.encoder.to(self.xpu_device).eval()
self.current_task_desc: Optional[str] = None
self.current_text_emb: Optional[torch.Tensor] = None
self.current_text_emb_xpu: Optional[torch.Tensor] = None
self.action_queue: List[np.ndarray] = []
self.stride_counter: int = 0
@classmethod
def from_pretrained(cls, pretrained_path: Union[str, Path] = ".", **kwargs) -> "ParvulusInference":
return cls(model_path=pretrained_path, **kwargs)
def preprocess_image(self, img: np.ndarray, target_device: Optional[torch.device] = None) -> torch.Tensor:
"""Preprocesses [H, W, 3] uint8 image -> [1, 3, res, res] tensor in [-1, 1]."""
dev = target_device or self.device
if self.flip_convention:
img = np.ascontiguousarray(img[::-1, ::-1])
if img.shape[0] != self.resolution or img.shape[1] != self.resolution:
img = np.array(Image.fromarray(img).resize((self.resolution, self.resolution), Image.BILINEAR))
if self.gamma is not None and self.gamma != 1.0:
img = cv2.LUT(img, self._gamma_lut)
elif self.auto_gamma:
mean_y = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY).mean()
if mean_y < 112.0:
img = cv2.LUT(img, self._gamma_lut)
t = torch.from_numpy(img).float() / 255.0
t = t.permute(2, 0, 1) * 2.0 - 1.0
return t.unsqueeze(0).to(dev)
def reset(self, task_description: Optional[str] = None, text_emb: Optional[torch.Tensor] = None):
"""Resets policy state for a new episode."""
if self.backend in ("parvulus", "pure_cuda", "cuda_c"):
if text_emb is not None:
vec = text_emb.detach().cpu().numpy()
self._c_engine.reset(task_description=task_description, task_vector=vec)
elif task_description is not None:
# If model with text_encoder is loaded, encode on the fly to support perturbations
if hasattr(self, "model") and hasattr(self.model, "text_encoder") and self.model.text_encoder is not None:
with torch.no_grad():
vec = self.model.text_encoder([task_description]).detach().cpu().numpy()[0]
self._c_engine.reset(task_description=task_description, task_vector=vec)
else:
self._c_engine.reset(task_description=task_description)
else:
self._c_engine.reset()
return
elif self.backend == "cuda_graph":
self._c_engine.reset(task_description)
return
self.current_task_desc = task_description
if text_emb is not None:
self.current_text_emb = text_emb if text_emb.dim() == 2 else text_emb.unsqueeze(0).to(self.device)
elif task_description is not None:
with torch.no_grad():
self.current_text_emb = self.model.text_encoder([task_description]).to(self.device)
else:
self.current_text_emb = None
if self.is_hybrid and self.current_text_emb is not None:
self.current_text_emb_xpu = self.current_text_emb.to(self.xpu_device)
self.action_queue.clear()
self.stride_counter = 0
@torch.no_grad()
def predict_chunk(
self,
agentview_img: np.ndarray,
wrist_img: np.ndarray,
task_description: Optional[str] = None,
denormalize: bool = True,
) -> np.ndarray:
"""Predicts full 8-step continuous action chunk [8, 7]."""
if self.backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"):
return self._c_engine.predict_chunk(agentview_img, wrist_img, task_description=task_description)
if task_description is not None:
self.reset(task_description)
elif self.current_text_emb is None:
raise ValueError("Call reset(task_description=...) before predicting actions!")
if self.is_hybrid:
agent_t = self.preprocess_image(agentview_img, target_device=self.xpu_device)
wrist_t = self.preprocess_image(wrist_img, target_device=self.xpu_device)
obs_xpu = torch.cat([agent_t, wrist_t], dim=1)
_, z_vis_xpu = self.model.encoder(obs_xpu, self.current_text_emb_xpu)
torch.xpu.synchronize()
z_vis = z_vis_xpu.to(self.device)
z = torch.randn(1, self.chunk_size, self.model.action_dim, device=self.device)
kv_layers = self.model._get_kv_layers()
kv_caches = [layer.precompute_kv(z_vis) for layer in kv_layers]
dt = 1.0 / self.num_ode_steps
for step in range(self.num_ode_steps):
sigma = torch.tensor([step * dt], device=self.device)
cond = self.current_text_emb + self.model.action_t_mlp(sigma)
h = self.model.action_decoder.action_in_proj(z) + self.model.action_decoder.pos_emb[:, :self.chunk_size, :]
for layer, (k_v, v_v) in zip(kv_layers, kv_caches):
h = layer.forward_with_kv(h, k_v, v_v, cond)
v_pred = self.model.action_decoder.out_head(self.model.action_decoder.final_norm(h))
z = z + v_pred * dt
chunk_norm = z.clamp(-1.0, 1.0)
else:
agent_t = self.preprocess_image(agentview_img)
wrist_t = self.preprocess_image(wrist_img)
obs = torch.cat([agent_t, wrist_t], dim=1)
chunk_norm = self.model.predict_action_chunk(
obs,
text_emb=self.current_text_emb,
num_ode_steps=self.num_ode_steps,
use_kv_cache=self.use_kv_cache,
)
if denormalize:
chunk = denormalize_action(chunk_norm.squeeze(0), self.action_min, self.action_max)
else:
chunk = chunk_norm.squeeze(0)
return chunk.cpu().numpy()
def step(self, agentview_img: np.ndarray, wrist_img: np.ndarray) -> np.ndarray:
"""Returns single 7-DoF action, replanning every C=replan_stride steps."""
if self.backend in ("parvulus", "pure_cuda", "cuda_c", "cuda_graph"):
return self._c_engine.step(agentview_img, wrist_img)
if not self.action_queue or self.stride_counter >= self.replan_stride:
chunk = self.predict_chunk(agentview_img, wrist_img, denormalize=True)
self.action_queue = list(chunk[: self.replan_stride])
self.stride_counter = 0
action = self.action_queue.pop(0)
self.stride_counter += 1
return action
@torch.no_grad()
def rollout_step(
self,
agentview_img: np.ndarray,
wrist_img: np.ndarray,
task_description: Optional[str] = None,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Auxiliary world-model step: predicts action chunk AND next dual-camera observation.
Returns:
action_chunk: [8, 7] float32 action chunk.
next_agent_img: [256, 256, 3] uint8 predicted agentview.
next_wrist_img: [256, 256, 3] uint8 predicted wristview.
"""
action_chunk = self.predict_chunk(agentview_img, wrist_img, task_description=task_description)
agent_t = self.preprocess_image(agentview_img)
wrist_t = self.preprocess_image(wrist_img)
obs = torch.cat([agent_t, wrist_t], dim=1)
pred_obs = self.model.predict_next_frame(obs, text_emb=self.current_text_emb, num_ode_steps=self.num_ode_steps)
pred_obs = ((pred_obs.squeeze(0) + 1.0) / 2.0 * 255.0).clamp(0, 255).byte().permute(1, 2, 0).cpu().numpy()
next_agent = pred_obs[:, :, 0:3]
next_wrist = pred_obs[:, :, 3:6]
if self.flip_convention:
next_agent = np.ascontiguousarray(next_agent[::-1, ::-1])
next_wrist = np.ascontiguousarray(next_wrist[::-1, ::-1])
return action_chunk, next_agent, next_wrist
ParvulusInfer = ParvulusInference
__all__ = ["ParvulusInference", "ParvulusInfer"]