Parvulus / test_continuous_pipelined.py
aasivas's picture
Upload folder using huggingface_hub
1871410 verified
Raw History Blame Contribute Delete
7.43 kB
#!/usr/bin/env python3
"""Continuous pipelined execution benchmark for Parvulus comparing sequential vs overlapped inference."""
import argparse
from concurrent.futures import ThreadPoolExecutor
import os
import sys
import time
from pathlib import Path
from typing import Dict, List, Tuple
import numpy as np
import torch
PROJECT_ROOT = Path(__file__).resolve().parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path:
sys.path.insert(0, os.environ["LIBERO_ROOT"])
from libero.libero import benchmark
from libero.libero.envs import OffScreenRenderEnv
from libero.libero import get_libero_path
from infer import ParvulusInference
def evaluate_trial(
env: OffScreenRenderEnv,
pipeline: ParvulusInference,
task_desc: str,
init_state: np.ndarray,
seed: int,
mode: str = "sequential", # "sequential" or "pipelined"
max_steps: int = 500,
) -> Dict[str, float]:
"""Runs a single rollout in sequential or continuous pipelined mode."""
env.seed(seed)
torch.manual_seed(seed)
np.random.seed(seed)
obs = env.reset()
obs = env.set_init_state(init_state)
pipeline.reset(task_desc)
settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0])
for _ in range(5):
obs, _, _, _ = env.step(settle_noop)
done = False
step_count = 0
total_robot_idle_time = 0.0
num_replans = 0
wall_start = time.perf_counter()
if mode == "sequential":
pipeline.replan_stride = 8
while not done and step_count < max_steps:
t0 = time.perf_counter()
action = pipeline.step(obs["agentview_image"], obs["robot0_eye_in_hand_image"])
wait_time = time.perf_counter() - t0
if wait_time > 0.005:
total_robot_idle_time += wait_time
num_replans += 1
obs, reward, done, info = env.step(action)
step_count += 1
if done:
break
elif mode == "pipelined":
t_init = time.perf_counter()
agent_img = obs["agentview_image"]
wrist_img = obs["robot0_eye_in_hand_image"]
current_chunk = pipeline.predict_chunk(agent_img, wrist_img)
init_idle = time.perf_counter() - t_init
total_robot_idle_time += init_idle
num_replans += 1
executor = ThreadPoolExecutor(max_workers=1)
for a in current_chunk[0:4]:
obs, reward, done, _ = env.step(a)
step_count += 1
if done:
break
while not done and step_count < max_steps:
agent_img = obs["agentview_image"]
wrist_img = obs["robot0_eye_in_hand_image"]
future = executor.submit(pipeline.predict_chunk, agent_img, wrist_img)
for a in current_chunk[4:8]:
obs, reward, done, _ = env.step(a)
step_count += 1
if done:
break
if done:
break
tw0 = time.perf_counter()
current_chunk = future.result()
idle_wait = time.perf_counter() - tw0
total_robot_idle_time += idle_wait
num_replans += 1
for a in current_chunk[0:4]:
obs, reward, done, _ = env.step(a)
step_count += 1
if done:
break
executor.shutdown(wait=False)
wall_total = time.perf_counter() - wall_start
success = float(done)
fps = step_count / wall_total if wall_total > 0 else 0.0
return {
"success": success,
"steps": step_count,
"wall_time_s": wall_total,
"fps": fps,
"num_replans": num_replans,
"total_robot_idle_ms": total_robot_idle_time * 1000.0,
"avg_idle_per_replan_ms": (total_robot_idle_time * 1000.0) / max(1, num_replans),
"avg_idle_per_step_ms": (total_robot_idle_time * 1000.0) / max(1, step_count),
}
def main():
parser = argparse.ArgumentParser(description="Continuous Pipelined Benchmark")
parser.add_argument("--device", type=str, default="cuda:0")
parser.add_argument("--suite", type=str, default="libero_spatial")
parser.add_argument("--task-id", type=int, default=0)
parser.add_argument("--trials", type=int, default=3)
args = parser.parse_args()
os.environ["CUDA_VISIBLE_DEVICES"] = args.device.replace("cuda:", "")
os.environ["MUJOCO_GL"] = "egl"
print("=" * 75)
print(f" CONTINUOUS PIPELINED EXECUTION BENCHMARK ({args.suite}, Task #{args.task_id})")
print(f" Device: {args.device} | Trials: {args.trials}")
print("=" * 75)
pipeline = ParvulusInference.from_pretrained(
PROJECT_ROOT,
device="cuda:0",
num_ode_steps=10,
chunk_size=8,
)
bench_dict = benchmark.get_benchmark_dict()
suite = bench_dict[args.suite]()
task_obj = suite.get_task(args.task_id)
task_desc = task_obj.language
bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file
with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]):
init_states = suite.get_task_init_states(args.task_id)
env = OffScreenRenderEnv(
bddl_file_name=str(bddl_file),
camera_heights=256,
camera_widths=256,
)
modes = ["sequential", "pipelined"]
results = {m: [] for m in modes}
for mode in modes:
print(f"\nEvaluating mode: {mode.upper()}...")
for trial in range(args.trials):
state = init_states[trial]
seed = 42 + trial
res = evaluate_trial(env, pipeline, task_desc, state, seed=seed, mode=mode)
results[mode].append(res)
print(
f" Trial {trial+1}/{args.trials}: "
f"Success={bool(res['success'])} | "
f"Steps={res['steps']} | "
f"Wall={res['wall_time_s']:.2f}s | "
f"Rate={res['fps']:.1f} steps/s | "
f"Total Idle={res['total_robot_idle_ms']:.2f} ms | "
f"Idle/Replan={res['avg_idle_per_replan_ms']:.2f} ms"
)
env.close()
print("\n" + "=" * 75)
print(" CONTINUOUS PIPELINED EXECUTION - FINAL COMPARATIVE REPORT:")
print("=" * 75)
for mode in modes:
res_list = results[mode]
succ = np.mean([r["success"] for r in res_list]) * 100.0
mean_steps = np.mean([r["steps"] for r in res_list])
mean_fps = np.mean([r["fps"] for r in res_list])
mean_idle = np.mean([r["total_robot_idle_ms"] for r in res_list])
mean_idle_replan = np.mean([r["avg_idle_per_replan_ms"] for r in res_list])
mean_idle_step = np.mean([r["avg_idle_per_step_ms"] for r in res_list])
print(f"Mode: {mode.upper()}")
print(f" - Benchmark Success Rate: {succ:.1f}%")
print(f" - Mean Steps to Task Success: {mean_steps:.1f} steps")
print(f" - Actual Execution Frequency: {mean_fps:.2f} steps/sec")
print(f" - Robot Idle Wait per Replan: {mean_idle_replan:.2f} ms")
print(f" - Robot Idle Wait per Step: {mean_idle_step:.3f} ms")
print(f" - Total Robot Idle per Rollout: {mean_idle:.2f} ms")
print("=" * 75)
if __name__ == "__main__":
main()