Download eval_libero.py from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 7.8 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/eval_libero.py
- Command line
-
hf download hf://aasivas/Parvulus/eval_libero.py
-
curl -L -o eval_libero.py https://huggingface.co/aasivas/Parvulus/resolve/main/eval_libero.py
7.8 kB
| #!/usr/bin/env python3 | |
| """Single-task and multi-trial LIBERO evaluation runner for Parvulus. | |
| Evaluates the Parvulus policy against the canonical LIBERO benchmark suites. | |
| Supports saving rollout visualization MP4 videos and reporting step-level metrics. | |
| """ | |
| import argparse | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Optional, Tuple | |
| import imageio | |
| import numpy as np | |
| import torch | |
| SCRIPT_DIR = Path(__file__).resolve().parent | |
| if str(SCRIPT_DIR) not in sys.path: | |
| sys.path.insert(0, str(SCRIPT_DIR)) | |
| if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path: | |
| sys.path.insert(0, os.environ["LIBERO_ROOT"]) | |
| from infer import ParvulusInference | |
| try: | |
| from libero.libero import benchmark, get_libero_path | |
| from libero.libero.envs import OffScreenRenderEnv | |
| except ImportError as e: | |
| raise ImportError( | |
| f"Could not import LIBERO ({e}). Ensure LIBERO and its dependencies are installed." | |
| ) | |
| TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"] | |
| MAX_STEPS = { | |
| "libero_spatial": 500, | |
| "libero_object": 500, | |
| "libero_goal": 500, | |
| "libero_10": 600, | |
| } | |
| def run_trial( | |
| infer: ParvulusInference, | |
| env: OffScreenRenderEnv, | |
| task_desc: str, | |
| init_state: Optional[np.ndarray] = None, | |
| max_steps: int = 500, | |
| settle_steps: int = 5, | |
| save_video_path: Optional[str] = None, | |
| ) -> Tuple[bool, int, float]: | |
| """Runs a single simulation rollout trial.""" | |
| infer.reset(task_description=task_desc) | |
| obs = env.reset() | |
| if init_state is not None: | |
| obs = env.set_init_state(init_state) | |
| # Initial settling with open gripper | |
| settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]) | |
| for _ in range(settle_steps): | |
| obs, _, _, _ = env.step(settle_noop) | |
| frames = [] | |
| if save_video_path: | |
| frames.append(obs["agentview_image"][::-1]) | |
| success = False | |
| step_count = 0 | |
| t0 = time.time() | |
| while step_count < max_steps and not success: | |
| agent_img = obs["agentview_image"] | |
| wrist_img = obs["robot0_eye_in_hand_image"] | |
| action = infer.step(agent_img, wrist_img) | |
| obs, reward, done, info = env.step(action) | |
| step_count += 1 | |
| if save_video_path: | |
| frames.append(obs["agentview_image"][::-1]) | |
| if done: | |
| # Probe steps to verify stability | |
| for _ in range(10): | |
| obs, _, d, _ = env.step(action) | |
| if save_video_path: | |
| frames.append(obs["agentview_image"][::-1]) | |
| if d: | |
| done = True | |
| # Retreat steps | |
| retreat_act = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0]) | |
| for _ in range(15): | |
| obs, _, d, _ = env.step(retreat_act) | |
| if save_video_path: | |
| frames.append(obs["agentview_image"][::-1]) | |
| # End settle frames | |
| for _ in range(5): | |
| obs, _, d, _ = env.step(np.zeros(7)) | |
| if save_video_path: | |
| frames.append(obs["agentview_image"][::-1]) | |
| success = bool(done) | |
| break | |
| elapsed = time.time() - t0 | |
| if save_video_path and frames: | |
| Path(save_video_path).parent.mkdir(parents=True, exist_ok=True) | |
| imageio.mimwrite(save_video_path, frames, fps=20) | |
| print(f"Saved trial video: {save_video_path}") | |
| return success, step_count, elapsed | |
| def parse_args(): | |
| parser = argparse.ArgumentParser(description="Parvulus LIBERO Benchmark Evaluation") | |
| parser.add_argument("--suite", type=str, default="libero_spatial", choices=TASK_SUITES) | |
| parser.add_argument("--task_id", type=int, default=0, help="Task index in suite (default: 0)") | |
| parser.add_argument("--num_trials", type=int, default=1, help="Number of trials to evaluate") | |
| parser.add_argument("--seed", type=int, default=42, help="Random seed (default: 42)") | |
| parser.add_argument("--device", type=str, default="cuda", choices=["cuda", "cpu", "xpu", "auto"]) | |
| parser.add_argument("--backend", type=str, default="pytorch", choices=["pytorch", "cuda_c", "cuda_graph", "pure_cuda"], help="Inference backend engine") | |
| parser.add_argument("--use_kv_cache", action="store_true", default=True, help="Enable fast KV caching") | |
| parser.add_argument("--num_ode_steps", type=int, default=10, help="Euler integration steps") | |
| parser.add_argument("--chunk_size", type=int, default=8, help="Action chunk horizon") | |
| parser.add_argument("--replan_stride", type=int, default=8, help="Receding horizon replan stride") | |
| parser.add_argument("--resolution", type=int, default=256, help="Camera resolution") | |
| parser.add_argument("--save_video", action="store_true", help="Save rollout video as MP4") | |
| parser.add_argument("--output_dir", type=str, default="./eval_results", help="Directory to save videos/results") | |
| return parser.parse_args() | |
| def main(): | |
| args = parse_args() | |
| print("=" * 70) | |
| print(" PARVULUS Physical AI Model - LIBERO Evaluation") | |
| print("=" * 70) | |
| print(f" Suite: {args.suite}") | |
| print(f" Task ID: {args.task_id}") | |
| print(f" Trials: {args.num_trials}") | |
| print(f" Device: {args.device}") | |
| print(f" KV Cache: {args.use_kv_cache}") | |
| print(f" ODE Steps: {args.num_ode_steps}") | |
| print("=" * 70) | |
| # Initialize policy | |
| infer = ParvulusInference.from_pretrained( | |
| pretrained_path=str(SCRIPT_DIR), | |
| device=args.device, | |
| backend=args.backend, | |
| use_kv_cache=args.use_kv_cache, | |
| num_ode_steps=args.num_ode_steps, | |
| chunk_size=args.chunk_size, | |
| replan_stride=args.replan_stride, | |
| resolution=args.resolution, | |
| ) | |
| # Initialize task suite | |
| suite_dict = benchmark.get_benchmark_dict() | |
| if args.suite not in suite_dict: | |
| raise ValueError(f"Unknown suite {args.suite}. Available: {list(suite_dict.keys())}") | |
| suite = suite_dict[args.suite]() | |
| task = suite.get_task(args.task_id) | |
| task_desc = task.language | |
| print(f"\n[Task {args.task_id}]: \"{task_desc}\"") | |
| bddl_file = Path(get_libero_path("bddl_files")) / task.problem_folder / task.bddl_file | |
| env = OffScreenRenderEnv( | |
| bddl_file_name=str(bddl_file), | |
| camera_heights=args.resolution, | |
| camera_widths=args.resolution, | |
| ) | |
| with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]): | |
| init_states = suite.get_task_init_states(args.task_id) | |
| max_steps = MAX_STEPS.get(args.suite, 500) | |
| successes = 0 | |
| for trial in range(args.num_trials): | |
| env.seed(args.seed + trial) | |
| torch.manual_seed(args.seed + trial) | |
| init_state = init_states[trial % len(init_states)] | |
| video_path = None | |
| if args.save_video: | |
| video_path = os.path.join( | |
| args.output_dir, f"{args.suite}_task{args.task_id}_trial{trial}.mp4" | |
| ) | |
| print(f"\n--- Running Trial {trial + 1}/{args.num_trials} ---") | |
| succ, steps, elapsed = run_trial( | |
| infer=infer, | |
| env=env, | |
| task_desc=task_desc, | |
| init_state=init_state, | |
| max_steps=max_steps, | |
| save_video_path=video_path, | |
| ) | |
| status_str = "SUCCESS" if succ else "FAILURE" | |
| print(f" Result: {status_str}") | |
| print(f" Steps: {steps}/{max_steps}") | |
| print(f" Latency: {elapsed:.2f}s ({steps / max(1e-5, elapsed):.1f} steps/s)") | |
| if succ: | |
| successes += 1 | |
| env.close() | |
| print("\n" + "=" * 70) | |
| print(f" SUMMARY: {successes}/{args.num_trials} ({successes / args.num_trials * 100:.1f}%) SUCCESS RATE") | |
| print("=" * 70) | |
| if __name__ == "__main__": | |
| main() | |