Download test_continuous_pipelined.py from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 7.43 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/test_continuous_pipelined.py
- Command line
-
hf download hf://aasivas/Parvulus/test_continuous_pipelined.py
-
curl -L -o test_continuous_pipelined.py https://huggingface.co/aasivas/Parvulus/resolve/main/test_continuous_pipelined.py
7.43 kB
| #!/usr/bin/env python3 | |
| """Continuous pipelined execution benchmark for Parvulus comparing sequential vs overlapped inference.""" | |
| import argparse | |
| from concurrent.futures import ThreadPoolExecutor | |
| import os | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Dict, List, Tuple | |
| import numpy as np | |
| import torch | |
| PROJECT_ROOT = Path(__file__).resolve().parent | |
| if str(PROJECT_ROOT) not in sys.path: | |
| sys.path.insert(0, str(PROJECT_ROOT)) | |
| if "LIBERO_ROOT" in os.environ and os.environ["LIBERO_ROOT"] not in sys.path: | |
| sys.path.insert(0, os.environ["LIBERO_ROOT"]) | |
| from libero.libero import benchmark | |
| from libero.libero.envs import OffScreenRenderEnv | |
| from libero.libero import get_libero_path | |
| from infer import ParvulusInference | |
| def evaluate_trial( | |
| env: OffScreenRenderEnv, | |
| pipeline: ParvulusInference, | |
| task_desc: str, | |
| init_state: np.ndarray, | |
| seed: int, | |
| mode: str = "sequential", # "sequential" or "pipelined" | |
| max_steps: int = 500, | |
| ) -> Dict[str, float]: | |
| """Runs a single rollout in sequential or continuous pipelined mode.""" | |
| env.seed(seed) | |
| torch.manual_seed(seed) | |
| np.random.seed(seed) | |
| obs = env.reset() | |
| obs = env.set_init_state(init_state) | |
| pipeline.reset(task_desc) | |
| settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]) | |
| for _ in range(5): | |
| obs, _, _, _ = env.step(settle_noop) | |
| done = False | |
| step_count = 0 | |
| total_robot_idle_time = 0.0 | |
| num_replans = 0 | |
| wall_start = time.perf_counter() | |
| if mode == "sequential": | |
| pipeline.replan_stride = 8 | |
| while not done and step_count < max_steps: | |
| t0 = time.perf_counter() | |
| action = pipeline.step(obs["agentview_image"], obs["robot0_eye_in_hand_image"]) | |
| wait_time = time.perf_counter() - t0 | |
| if wait_time > 0.005: | |
| total_robot_idle_time += wait_time | |
| num_replans += 1 | |
| obs, reward, done, info = env.step(action) | |
| step_count += 1 | |
| if done: | |
| break | |
| elif mode == "pipelined": | |
| t_init = time.perf_counter() | |
| agent_img = obs["agentview_image"] | |
| wrist_img = obs["robot0_eye_in_hand_image"] | |
| current_chunk = pipeline.predict_chunk(agent_img, wrist_img) | |
| init_idle = time.perf_counter() - t_init | |
| total_robot_idle_time += init_idle | |
| num_replans += 1 | |
| executor = ThreadPoolExecutor(max_workers=1) | |
| for a in current_chunk[0:4]: | |
| obs, reward, done, _ = env.step(a) | |
| step_count += 1 | |
| if done: | |
| break | |
| while not done and step_count < max_steps: | |
| agent_img = obs["agentview_image"] | |
| wrist_img = obs["robot0_eye_in_hand_image"] | |
| future = executor.submit(pipeline.predict_chunk, agent_img, wrist_img) | |
| for a in current_chunk[4:8]: | |
| obs, reward, done, _ = env.step(a) | |
| step_count += 1 | |
| if done: | |
| break | |
| if done: | |
| break | |
| tw0 = time.perf_counter() | |
| current_chunk = future.result() | |
| idle_wait = time.perf_counter() - tw0 | |
| total_robot_idle_time += idle_wait | |
| num_replans += 1 | |
| for a in current_chunk[0:4]: | |
| obs, reward, done, _ = env.step(a) | |
| step_count += 1 | |
| if done: | |
| break | |
| executor.shutdown(wait=False) | |
| wall_total = time.perf_counter() - wall_start | |
| success = float(done) | |
| fps = step_count / wall_total if wall_total > 0 else 0.0 | |
| return { | |
| "success": success, | |
| "steps": step_count, | |
| "wall_time_s": wall_total, | |
| "fps": fps, | |
| "num_replans": num_replans, | |
| "total_robot_idle_ms": total_robot_idle_time * 1000.0, | |
| "avg_idle_per_replan_ms": (total_robot_idle_time * 1000.0) / max(1, num_replans), | |
| "avg_idle_per_step_ms": (total_robot_idle_time * 1000.0) / max(1, step_count), | |
| } | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Continuous Pipelined Benchmark") | |
| parser.add_argument("--device", type=str, default="cuda:0") | |
| parser.add_argument("--suite", type=str, default="libero_spatial") | |
| parser.add_argument("--task-id", type=int, default=0) | |
| parser.add_argument("--trials", type=int, default=3) | |
| args = parser.parse_args() | |
| os.environ["CUDA_VISIBLE_DEVICES"] = args.device.replace("cuda:", "") | |
| os.environ["MUJOCO_GL"] = "egl" | |
| print("=" * 75) | |
| print(f" CONTINUOUS PIPELINED EXECUTION BENCHMARK ({args.suite}, Task #{args.task_id})") | |
| print(f" Device: {args.device} | Trials: {args.trials}") | |
| print("=" * 75) | |
| pipeline = ParvulusInference.from_pretrained( | |
| PROJECT_ROOT, | |
| device="cuda:0", | |
| num_ode_steps=10, | |
| chunk_size=8, | |
| ) | |
| bench_dict = benchmark.get_benchmark_dict() | |
| suite = bench_dict[args.suite]() | |
| task_obj = suite.get_task(args.task_id) | |
| task_desc = task_obj.language | |
| bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file | |
| with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]): | |
| init_states = suite.get_task_init_states(args.task_id) | |
| env = OffScreenRenderEnv( | |
| bddl_file_name=str(bddl_file), | |
| camera_heights=256, | |
| camera_widths=256, | |
| ) | |
| modes = ["sequential", "pipelined"] | |
| results = {m: [] for m in modes} | |
| for mode in modes: | |
| print(f"\nEvaluating mode: {mode.upper()}...") | |
| for trial in range(args.trials): | |
| state = init_states[trial] | |
| seed = 42 + trial | |
| res = evaluate_trial(env, pipeline, task_desc, state, seed=seed, mode=mode) | |
| results[mode].append(res) | |
| print( | |
| f" Trial {trial+1}/{args.trials}: " | |
| f"Success={bool(res['success'])} | " | |
| f"Steps={res['steps']} | " | |
| f"Wall={res['wall_time_s']:.2f}s | " | |
| f"Rate={res['fps']:.1f} steps/s | " | |
| f"Total Idle={res['total_robot_idle_ms']:.2f} ms | " | |
| f"Idle/Replan={res['avg_idle_per_replan_ms']:.2f} ms" | |
| ) | |
| env.close() | |
| print("\n" + "=" * 75) | |
| print(" CONTINUOUS PIPELINED EXECUTION - FINAL COMPARATIVE REPORT:") | |
| print("=" * 75) | |
| for mode in modes: | |
| res_list = results[mode] | |
| succ = np.mean([r["success"] for r in res_list]) * 100.0 | |
| mean_steps = np.mean([r["steps"] for r in res_list]) | |
| mean_fps = np.mean([r["fps"] for r in res_list]) | |
| mean_idle = np.mean([r["total_robot_idle_ms"] for r in res_list]) | |
| mean_idle_replan = np.mean([r["avg_idle_per_replan_ms"] for r in res_list]) | |
| mean_idle_step = np.mean([r["avg_idle_per_step_ms"] for r in res_list]) | |
| print(f"Mode: {mode.upper()}") | |
| print(f" - Benchmark Success Rate: {succ:.1f}%") | |
| print(f" - Mean Steps to Task Success: {mean_steps:.1f} steps") | |
| print(f" - Actual Execution Frequency: {mean_fps:.2f} steps/sec") | |
| print(f" - Robot Idle Wait per Replan: {mean_idle_replan:.2f} ms") | |
| print(f" - Robot Idle Wait per Step: {mean_idle_step:.3f} ms") | |
| print(f" - Total Robot Idle per Rollout: {mean_idle:.2f} ms") | |
| print("=" * 75) | |
| if __name__ == "__main__": | |
| main() | |