#!/usr/bin/env python3 """Continuous pipelined execution benchmark for Parvulus comparing sequential vs overlapped inference.""" import argparse from concurrent.futures import ThreadPoolExecutor import os import sys import time from pathlib import Path from typing import Dict, List, Tuple import numpy as np import torch PROJECT_ROOT = Path(__file__).resolve().parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path: sys.path.insert(0, os.environ["LIBERO_ROOT"]) from libero.libero import benchmark from libero.libero.envs import OffScreenRenderEnv from libero.libero import get_libero_path from infer import ParvulusInference def evaluate_trial( env: OffScreenRenderEnv, pipeline: ParvulusInference, task_desc: str, init_state: np.ndarray, seed: int, mode: str = "sequential", # "sequential" or "pipelined" max_steps: int = 500, ) -> Dict[str, float]: """Runs a single rollout in sequential or continuous pipelined mode.""" env.seed(seed) torch.manual_seed(seed) np.random.seed(seed) obs = env.reset() obs = env.set_init_state(init_state) pipeline.reset(task_desc) settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]) for _ in range(5): obs, _, _, _ = env.step(settle_noop) done = False step_count = 0 total_robot_idle_time = 0.0 num_replans = 0 wall_start = time.perf_counter() if mode == "sequential": pipeline.replan_stride = 8 while not done and step_count < max_steps: t0 = time.perf_counter() action = pipeline.step(obs["agentview_image"], obs["robot0_eye_in_hand_image"]) wait_time = time.perf_counter() - t0 if wait_time > 0.005: total_robot_idle_time += wait_time num_replans += 1 obs, reward, done, info = env.step(action) step_count += 1 if done: break elif mode == "pipelined": t_init = time.perf_counter() agent_img = obs["agentview_image"] wrist_img = obs["robot0_eye_in_hand_image"] current_chunk = pipeline.predict_chunk(agent_img, wrist_img) init_idle = time.perf_counter() - t_init total_robot_idle_time += init_idle num_replans += 1 executor = ThreadPoolExecutor(max_workers=1) for a in current_chunk[0:4]: obs, reward, done, _ = env.step(a) step_count += 1 if done: break while not done and step_count < max_steps: agent_img = obs["agentview_image"] wrist_img = obs["robot0_eye_in_hand_image"] future = executor.submit(pipeline.predict_chunk, agent_img, wrist_img) for a in current_chunk[4:8]: obs, reward, done, _ = env.step(a) step_count += 1 if done: break if done: break tw0 = time.perf_counter() current_chunk = future.result() idle_wait = time.perf_counter() - tw0 total_robot_idle_time += idle_wait num_replans += 1 for a in current_chunk[0:4]: obs, reward, done, _ = env.step(a) step_count += 1 if done: break executor.shutdown(wait=False) wall_total = time.perf_counter() - wall_start success = float(done) fps = step_count / wall_total if wall_total > 0 else 0.0 return { "success": success, "steps": step_count, "wall_time_s": wall_total, "fps": fps, "num_replans": num_replans, "total_robot_idle_ms": total_robot_idle_time * 1000.0, "avg_idle_per_replan_ms": (total_robot_idle_time * 1000.0) / max(1, num_replans), "avg_idle_per_step_ms": (total_robot_idle_time * 1000.0) / max(1, step_count), } def main(): parser = argparse.ArgumentParser(description="Continuous Pipelined Benchmark") parser.add_argument("--device", type=str, default="cuda:0") parser.add_argument("--suite", type=str, default="libero_spatial") parser.add_argument("--task-id", type=int, default=0) parser.add_argument("--trials", type=int, default=3) args = parser.parse_args() os.environ["CUDA_VISIBLE_DEVICES"] = args.device.replace("cuda:", "") os.environ["MUJOCO_GL"] = "egl" print("=" * 75) print(f" CONTINUOUS PIPELINED EXECUTION BENCHMARK ({args.suite}, Task #{args.task_id})") print(f" Device: {args.device} | Trials: {args.trials}") print("=" * 75) pipeline = ParvulusInference.from_pretrained( PROJECT_ROOT, device="cuda:0", num_ode_steps=10, chunk_size=8, ) bench_dict = benchmark.get_benchmark_dict() suite = bench_dict[args.suite]() task_obj = suite.get_task(args.task_id) task_desc = task_obj.language bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]): init_states = suite.get_task_init_states(args.task_id) env = OffScreenRenderEnv( bddl_file_name=str(bddl_file), camera_heights=256, camera_widths=256, ) modes = ["sequential", "pipelined"] results = {m: [] for m in modes} for mode in modes: print(f"\nEvaluating mode: {mode.upper()}...") for trial in range(args.trials): state = init_states[trial] seed = 42 + trial res = evaluate_trial(env, pipeline, task_desc, state, seed=seed, mode=mode) results[mode].append(res) print( f" Trial {trial+1}/{args.trials}: " f"Success={bool(res['success'])} | " f"Steps={res['steps']} | " f"Wall={res['wall_time_s']:.2f}s | " f"Rate={res['fps']:.1f} steps/s | " f"Total Idle={res['total_robot_idle_ms']:.2f} ms | " f"Idle/Replan={res['avg_idle_per_replan_ms']:.2f} ms" ) env.close() print("\n" + "=" * 75) print(" CONTINUOUS PIPELINED EXECUTION - FINAL COMPARATIVE REPORT:") print("=" * 75) for mode in modes: res_list = results[mode] succ = np.mean([r["success"] for r in res_list]) * 100.0 mean_steps = np.mean([r["steps"] for r in res_list]) mean_fps = np.mean([r["fps"] for r in res_list]) mean_idle = np.mean([r["total_robot_idle_ms"] for r in res_list]) mean_idle_replan = np.mean([r["avg_idle_per_replan_ms"] for r in res_list]) mean_idle_step = np.mean([r["avg_idle_per_step_ms"] for r in res_list]) print(f"Mode: {mode.upper()}") print(f" - Benchmark Success Rate: {succ:.1f}%") print(f" - Mean Steps to Task Success: {mean_steps:.1f} steps") print(f" - Actual Execution Frequency: {mean_fps:.2f} steps/sec") print(f" - Robot Idle Wait per Replan: {mean_idle_replan:.2f} ms") print(f" - Robot Idle Wait per Step: {mean_idle_step:.3f} ms") print(f" - Total Robot Idle per Rollout: {mean_idle:.2f} ms") print("=" * 75) if __name__ == "__main__": main()