#!/usr/bin/env python3 """Single-task and multi-trial LIBERO evaluation runner for Parvulus. Evaluates the Parvulus policy against the canonical LIBERO benchmark suites. Supports saving rollout visualization MP4 videos and reporting step-level metrics. """ import argparse import os import sys import time from pathlib import Path from typing import Optional, Tuple import imageio import numpy as np import torch SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path: sys.path.insert(0, os.environ["LIBERO_ROOT"]) from infer import ParvulusInference try: from libero.libero import benchmark, get_libero_path from libero.libero.envs import OffScreenRenderEnv except ImportError as e: raise ImportError( f"Could not import LIBERO ({e}). Ensure LIBERO and its dependencies are installed." ) TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"] MAX_STEPS = { "libero_spatial": 500, "libero_object": 500, "libero_goal": 500, "libero_10": 600, } def run_trial( infer: ParvulusInference, env: OffScreenRenderEnv, task_desc: str, init_state: Optional[np.ndarray] = None, max_steps: int = 500, settle_steps: int = 5, save_video_path: Optional[str] = None, ) -> Tuple[bool, int, float]: """Runs a single simulation rollout trial.""" infer.reset(task_description=task_desc) obs = env.reset() if init_state is not None: obs = env.set_init_state(init_state) # Initial settling with open gripper settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]) for _ in range(settle_steps): obs, _, _, _ = env.step(settle_noop) frames = [] if save_video_path: frames.append(obs["agentview_image"][::-1]) success = False step_count = 0 t0 = time.time() while step_count < max_steps and not success: agent_img = obs["agentview_image"] wrist_img = obs["robot0_eye_in_hand_image"] action = infer.step(agent_img, wrist_img) obs, reward, done, info = env.step(action) step_count += 1 if save_video_path: frames.append(obs["agentview_image"][::-1]) if done: # Probe steps to verify stability for _ in range(10): obs, _, d, _ = env.step(action) if save_video_path: frames.append(obs["agentview_image"][::-1]) if d: done = True # Retreat steps retreat_act = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0]) for _ in range(15): obs, _, d, _ = env.step(retreat_act) if save_video_path: frames.append(obs["agentview_image"][::-1]) # End settle frames for _ in range(5): obs, _, d, _ = env.step(np.zeros(7)) if save_video_path: frames.append(obs["agentview_image"][::-1]) success = bool(done) break elapsed = time.time() - t0 if save_video_path and frames: Path(save_video_path).parent.mkdir(parents=True, exist_ok=True) imageio.mimwrite(save_video_path, frames, fps=20) print(f"Saved trial video: {save_video_path}") return success, step_count, elapsed def parse_args(): parser = argparse.ArgumentParser(description="Parvulus LIBERO Benchmark Evaluation") parser.add_argument("--suite", type=str, default="libero_spatial", choices=TASK_SUITES) parser.add_argument("--task_id", type=int, default=0, help="Task index in suite (default: 0)") parser.add_argument("--num_trials", type=int, default=1, help="Number of trials to evaluate") parser.add_argument("--seed", type=int, default=42, help="Random seed (default: 42)") parser.add_argument("--device", type=str, default="cuda", choices=["cuda", "cpu", "xpu", "auto"]) parser.add_argument("--backend", type=str, default="pytorch", choices=["pytorch", "cuda_c", "cuda_graph", "pure_cuda"], help="Inference backend engine") parser.add_argument("--use_kv_cache", action="store_true", default=True, help="Enable fast KV caching") parser.add_argument("--num_ode_steps", type=int, default=10, help="Euler integration steps") parser.add_argument("--chunk_size", type=int, default=8, help="Action chunk horizon") parser.add_argument("--replan_stride", type=int, default=8, help="Receding horizon replan stride") parser.add_argument("--resolution", type=int, default=256, help="Camera resolution") parser.add_argument("--save_video", action="store_true", help="Save rollout video as MP4") parser.add_argument("--output_dir", type=str, default="./eval_results", help="Directory to save videos/results") return parser.parse_args() def main(): args = parse_args() print("=" * 70) print(" PARVULUS Physical AI Model - LIBERO Evaluation") print("=" * 70) print(f" Suite: {args.suite}") print(f" Task ID: {args.task_id}") print(f" Trials: {args.num_trials}") print(f" Device: {args.device}") print(f" KV Cache: {args.use_kv_cache}") print(f" ODE Steps: {args.num_ode_steps}") print("=" * 70) # Initialize policy infer = ParvulusInference.from_pretrained( pretrained_path=str(SCRIPT_DIR), device=args.device, backend=args.backend, use_kv_cache=args.use_kv_cache, num_ode_steps=args.num_ode_steps, chunk_size=args.chunk_size, replan_stride=args.replan_stride, resolution=args.resolution, ) # Initialize task suite suite_dict = benchmark.get_benchmark_dict() if args.suite not in suite_dict: raise ValueError(f"Unknown suite {args.suite}. Available: {list(suite_dict.keys())}") suite = suite_dict[args.suite]() task = suite.get_task(args.task_id) task_desc = task.language print(f"\n[Task {args.task_id}]: \"{task_desc}\"") bddl_file = Path(get_libero_path("bddl_files")) / task.problem_folder / task.bddl_file env = OffScreenRenderEnv( bddl_file_name=str(bddl_file), camera_heights=args.resolution, camera_widths=args.resolution, ) with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]): init_states = suite.get_task_init_states(args.task_id) max_steps = MAX_STEPS.get(args.suite, 500) successes = 0 for trial in range(args.num_trials): env.seed(args.seed + trial) torch.manual_seed(args.seed + trial) init_state = init_states[trial % len(init_states)] video_path = None if args.save_video: video_path = os.path.join( args.output_dir, f"{args.suite}_task{args.task_id}_trial{trial}.mp4" ) print(f"\n--- Running Trial {trial + 1}/{args.num_trials} ---") succ, steps, elapsed = run_trial( infer=infer, env=env, task_desc=task_desc, init_state=init_state, max_steps=max_steps, save_video_path=video_path, ) status_str = "SUCCESS" if succ else "FAILURE" print(f" Result: {status_str}") print(f" Steps: {steps}/{max_steps}") print(f" Latency: {elapsed:.2f}s ({steps / max(1e-5, elapsed):.1f} steps/s)") if succ: successes += 1 env.close() print("\n" + "=" * 70) print(f" SUMMARY: {successes}/{args.num_trials} ({successes / args.num_trials * 100:.1f}%) SUCCESS RATE") print("=" * 70) if __name__ == "__main__": main()