Parvulus / eval_libero.py
aasivas's picture
Upload folder using huggingface_hub
1871410 verified
Raw History Blame Contribute Delete
7.8 kB
#!/usr/bin/env python3
"""Single-task and multi-trial LIBERO evaluation runner for Parvulus.
Evaluates the Parvulus policy against the canonical LIBERO benchmark suites.
Supports saving rollout visualization MP4 videos and reporting step-level metrics.
"""
import argparse
import os
import sys
import time
from pathlib import Path
from typing import Optional, Tuple
import imageio
import numpy as np
import torch
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path:
sys.path.insert(0, os.environ["LIBERO_ROOT"])
from infer import ParvulusInference
try:
from libero.libero import benchmark, get_libero_path
from libero.libero.envs import OffScreenRenderEnv
except ImportError as e:
raise ImportError(
f"Could not import LIBERO ({e}). Ensure LIBERO and its dependencies are installed."
)
TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"]
MAX_STEPS = {
"libero_spatial": 500,
"libero_object": 500,
"libero_goal": 500,
"libero_10": 600,
}
def run_trial(
infer: ParvulusInference,
env: OffScreenRenderEnv,
task_desc: str,
init_state: Optional[np.ndarray] = None,
max_steps: int = 500,
settle_steps: int = 5,
save_video_path: Optional[str] = None,
) -> Tuple[bool, int, float]:
"""Runs a single simulation rollout trial."""
infer.reset(task_description=task_desc)
obs = env.reset()
if init_state is not None:
obs = env.set_init_state(init_state)
# Initial settling with open gripper
settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
for _ in range(settle_steps):
obs, _, _, _ = env.step(settle_noop)
frames = []
if save_video_path:
frames.append(obs["agentview_image"][::-1])
success = False
step_count = 0
t0 = time.time()
while step_count < max_steps and not success:
agent_img = obs["agentview_image"]
wrist_img = obs["robot0_eye_in_hand_image"]
action = infer.step(agent_img, wrist_img)
obs, reward, done, info = env.step(action)
step_count += 1
if save_video_path:
frames.append(obs["agentview_image"][::-1])
if done:
# Probe steps to verify stability
for _ in range(10):
obs, _, d, _ = env.step(action)
if save_video_path:
frames.append(obs["agentview_image"][::-1])
if d:
done = True
# Retreat steps
retreat_act = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0])
for _ in range(15):
obs, _, d, _ = env.step(retreat_act)
if save_video_path:
frames.append(obs["agentview_image"][::-1])
# End settle frames
for _ in range(5):
obs, _, d, _ = env.step(np.zeros(7))
if save_video_path:
frames.append(obs["agentview_image"][::-1])
success = bool(done)
break
elapsed = time.time() - t0
if save_video_path and frames:
Path(save_video_path).parent.mkdir(parents=True, exist_ok=True)
imageio.mimwrite(save_video_path, frames, fps=20)
print(f"Saved trial video: {save_video_path}")
return success, step_count, elapsed
def parse_args():
parser = argparse.ArgumentParser(description="Parvulus LIBERO Benchmark Evaluation")
parser.add_argument("--suite", type=str, default="libero_spatial", choices=TASK_SUITES)
parser.add_argument("--task_id", type=int, default=0, help="Task index in suite (default: 0)")
parser.add_argument("--num_trials", type=int, default=1, help="Number of trials to evaluate")
parser.add_argument("--seed", type=int, default=42, help="Random seed (default: 42)")
parser.add_argument("--device", type=str, default="cuda", choices=["cuda", "cpu", "xpu", "auto"])
parser.add_argument("--backend", type=str, default="pytorch", choices=["pytorch", "cuda_c", "cuda_graph", "pure_cuda"], help="Inference backend engine")
parser.add_argument("--use_kv_cache", action="store_true", default=True, help="Enable fast KV caching")
parser.add_argument("--num_ode_steps", type=int, default=10, help="Euler integration steps")
parser.add_argument("--chunk_size", type=int, default=8, help="Action chunk horizon")
parser.add_argument("--replan_stride", type=int, default=8, help="Receding horizon replan stride")
parser.add_argument("--resolution", type=int, default=256, help="Camera resolution")
parser.add_argument("--save_video", action="store_true", help="Save rollout video as MP4")
parser.add_argument("--output_dir", type=str, default="./eval_results", help="Directory to save videos/results")
return parser.parse_args()
def main():
args = parse_args()
print("=" * 70)
print(" PARVULUS Physical AI Model - LIBERO Evaluation")
print("=" * 70)
print(f" Suite: {args.suite}")
print(f" Task ID: {args.task_id}")
print(f" Trials: {args.num_trials}")
print(f" Device: {args.device}")
print(f" KV Cache: {args.use_kv_cache}")
print(f" ODE Steps: {args.num_ode_steps}")
print("=" * 70)
# Initialize policy
infer = ParvulusInference.from_pretrained(
pretrained_path=str(SCRIPT_DIR),
device=args.device,
backend=args.backend,
use_kv_cache=args.use_kv_cache,
num_ode_steps=args.num_ode_steps,
chunk_size=args.chunk_size,
replan_stride=args.replan_stride,
resolution=args.resolution,
)
# Initialize task suite
suite_dict = benchmark.get_benchmark_dict()
if args.suite not in suite_dict:
raise ValueError(f"Unknown suite {args.suite}. Available: {list(suite_dict.keys())}")
suite = suite_dict[args.suite]()
task = suite.get_task(args.task_id)
task_desc = task.language
print(f"\n[Task {args.task_id}]: \"{task_desc}\"")
bddl_file = Path(get_libero_path("bddl_files")) / task.problem_folder / task.bddl_file
env = OffScreenRenderEnv(
bddl_file_name=str(bddl_file),
camera_heights=args.resolution,
camera_widths=args.resolution,
)
with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]):
init_states = suite.get_task_init_states(args.task_id)
max_steps = MAX_STEPS.get(args.suite, 500)
successes = 0
for trial in range(args.num_trials):
env.seed(args.seed + trial)
torch.manual_seed(args.seed + trial)
init_state = init_states[trial % len(init_states)]
video_path = None
if args.save_video:
video_path = os.path.join(
args.output_dir, f"{args.suite}_task{args.task_id}_trial{trial}.mp4"
)
print(f"\n--- Running Trial {trial + 1}/{args.num_trials} ---")
succ, steps, elapsed = run_trial(
infer=infer,
env=env,
task_desc=task_desc,
init_state=init_state,
max_steps=max_steps,
save_video_path=video_path,
)
status_str = "SUCCESS" if succ else "FAILURE"
print(f" Result: {status_str}")
print(f" Steps: {steps}/{max_steps}")
print(f" Latency: {elapsed:.2f}s ({steps / max(1e-5, elapsed):.1f} steps/s)")
if succ:
successes += 1
env.close()
print("\n" + "=" * 70)
print(f" SUMMARY: {successes}/{args.num_trials} ({successes / args.num_trials * 100:.1f}%) SUCCESS RATE")
print("=" * 70)
if __name__ == "__main__":
main()