#!/usr/bin/env python3 """High-Performance Parallel Benchmark Evaluator for Parvulus across LIBERO. Runs all 40 canonical tasks across 4 suites (50 trials each = 2,000 trials) distributed dynamically across multiple GPUs and worker processes. Uses atomic lock queue for dynamic load balancing and fault tolerance. """ import argparse import json import os import subprocess import sys import time from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import matplotlib.pyplot as plt import numpy as np TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"] MAX_STEPS = { "libero_spatial": 500, "libero_object": 500, "libero_goal": 500, "libero_10": 600, } SCRIPT_DIR = Path(__file__).resolve().parent PROJECT_ROOT = SCRIPT_DIR LIBERO_ROOT = Path(os.environ["LIBERO_ROOT"]) if "LIBERO_ROOT" in os.environ else None # --------------------------------------------------------------------------- # Worker process entrypoint # --------------------------------------------------------------------------- def run_worker( worker_id: int, gpu_id: int, queue_dir: Path, results_dir: Path, tasks_meta_file: Path, num_trials: int, chunk_size: int, replan_stride: int, num_ode_steps: int, resolution: int, seed_offset: int, backend: str = "cuda_c", ): """Worker process: loops over dynamic task queue and executes rollouts.""" # Ensure correct environment variables before importing heavy frameworks os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id) os.environ["MUJOCO_GL"] = "egl" os.environ["TOKENIZERS_PARALLELISM"] = "false" if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) if LIBERO_ROOT and LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path: sys.path.insert(0, str(LIBERO_ROOT)) import torch from infer import ParvulusInference from libero.libero import benchmark, get_libero_path from libero.libero.envs import OffScreenRenderEnv # Load tasks metadata with open(tasks_meta_file, "r") as f: tasks = json.load(f) device = "cuda:0" print(f"[Worker {worker_id} (GPU {gpu_id})] Initializing Parvulus policy on {device} (backend={backend})...", flush=True) infer = ParvulusInference.from_pretrained( pretrained_path=str(SCRIPT_DIR), device=device, backend=backend, use_kv_cache=True, num_ode_steps=num_ode_steps, chunk_size=chunk_size, replan_stride=replan_stride, resolution=resolution, ) bench_dict = benchmark.get_benchmark_dict() loaded_suites: Dict[str, Any] = {} settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]) retreat_action = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0]) print(f"[Worker {worker_id} (GPU {gpu_id})] Ready. Polling task queue...", flush=True) while True: # Atomic claim of next unclaimed task claimed_idx = None for idx in range(len(tasks)): result_file = results_dir / f"result_task_{idx:02d}.json" if result_file.exists(): continue lock_file = queue_dir / f"task_{idx:02d}.lock" try: fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY) with os.fdopen(fd, "w") as lf: lf.write(f"worker_{worker_id}_pid_{os.getpid()}\n") claimed_idx = idx break except FileExistsError: continue if claimed_idx is None: # Check if all tasks have results all_done = all((results_dir / f"result_task_{i:02d}.json").exists() for i in range(len(tasks))) if all_done: break time.sleep(1.0) continue task_info = tasks[claimed_idx] suite_name = task_info["suite_name"] task_id = task_info["task_id"] task_desc = task_info["language"] task_name = task_info["task_name"] max_steps = MAX_STEPS.get(suite_name, 500) print( f"[Worker {worker_id} (GPU {gpu_id})] Claimed Task {claimed_idx:02d} " f"[{suite_name} #{task_id}]: '{task_desc[:40]}...'", flush=True, ) if suite_name not in loaded_suites: loaded_suites[suite_name] = bench_dict[suite_name]() suite = loaded_suites[suite_name] task_obj = suite.get_task(task_id) # Load canonical initial states with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]): init_states = suite.get_task_init_states(task_id) bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file env = OffScreenRenderEnv( bddl_file_name=str(bddl_file), camera_heights=resolution, camera_widths=resolution, ) infer.reset(task_description=task_desc) trial_records = [] task_successes = 0 task_t0 = time.time() for trial in range(num_trials): trial_seed = seed_offset + claimed_idx * 1000 + trial env.seed(trial_seed) torch.manual_seed(trial_seed) np.random.seed(trial_seed) obs = env.reset() init_state = init_states[trial % len(init_states)] obs = env.set_init_state(init_state) infer.reset(task_description=task_desc) # Settle for _ in range(5): obs, _, _, _ = env.step(settle_noop) success = False step_count = 0 trial_t0 = time.time() while step_count < max_steps and not success: agent_img = obs["agentview_image"] wrist_img = obs["robot0_eye_in_hand_image"] action = infer.step(agent_img, wrist_img) obs, reward, done, info = env.step(action) step_count += 1 if done: # Probe for _ in range(10): obs, _, d, _ = env.step(action) if d: done = True # Retreat for _ in range(15): obs, _, d, _ = env.step(retreat_action) # Settle for _ in range(5): obs, _, d, _ = env.step(np.zeros(7)) success = bool(done) break trial_elapsed = time.time() - trial_t0 if success: task_successes += 1 trial_records.append({ "trial": trial, "success": success, "steps": step_count, "elapsed": round(trial_elapsed, 2), }) if (trial + 1) % 10 == 0 or trial == num_trials - 1: cur_rate = task_successes / (trial + 1) * 100 print( f" [Worker {worker_id} | Task {claimed_idx:02d}] " f"Trial {trial+1:02d}/{num_trials}: {task_successes}/{trial+1} ({cur_rate:.1f}%) " f"[{trial_elapsed:.2f}s, {step_count} steps]", flush=True, ) env.close() task_elapsed = time.time() - task_t0 # Save task result atomically result_payload = { "global_task_idx": claimed_idx, "suite_name": suite_name, "task_id": task_id, "task_name": task_name, "task_desc": task_desc, "num_trials": num_trials, "successes": task_successes, "success_rate": task_successes / num_trials, "task_elapsed_seconds": round(task_elapsed, 2), "worker_id": worker_id, "gpu_id": gpu_id, "trials": trial_records, } tmp_result = results_dir / f"result_task_{claimed_idx:02d}.tmp" final_result = results_dir / f"result_task_{claimed_idx:02d}.json" with open(tmp_result, "w") as f: json.dump(result_payload, f, indent=2) os.replace(tmp_result, final_result) print( f"[Worker {worker_id} (GPU {gpu_id})] COMPLETED Task {claimed_idx:02d} " f"({suite_name} #{task_id}) -> {task_successes}/{num_trials} " f"({task_successes/num_trials*100:.1f}%) in {task_elapsed:.1f}s", flush=True, ) print(f"[Worker {worker_id} (GPU {gpu_id})] Exiting cleanly.", flush=True) # --------------------------------------------------------------------------- # Plotting helper # --------------------------------------------------------------------------- def plot_results(suite_summaries: Dict[str, Dict], overall_rate: float, total_trials: int, save_path: str): suites = list(suite_summaries.keys()) labels = [s.replace("libero_", "").upper() for s in suites] scores = [suite_summaries[s]["rate"] * 100 for s in suites] if all(s in suite_summaries for s in ["libero_spatial", "libero_object", "libero_goal"]): s3_succ = sum(suite_summaries[s]["successes"] for s in ["libero_spatial", "libero_object", "libero_goal"]) s3_trials = sum(suite_summaries[s]["trials"] for s in ["libero_spatial", "libero_object", "libero_goal"]) labels.append("3-SUITE") scores.append((s3_succ / s3_trials) * 100) labels.append("OVERALL") scores.append(overall_rate * 100) colors = ["#4C72B0", "#55A868", "#C44E52", "#8172B3", "#CCB974", "#64B5CD"][:len(labels)] plt.figure(figsize=(max(8, len(labels) * 1.5), 5.5)) bars = plt.bar(labels, scores, color=colors, width=0.55, edgecolor="black", linewidth=1.2) plt.axhline(90, color="red", linestyle="--", linewidth=1.5, label="Gold Reference (90%)") plt.ylim(0, 105) plt.ylabel("Success Rate (%)", fontsize=12, fontweight="bold") plt.title(f"PARVULUS LIBERO Benchmark Evaluation ({total_trials} Trials)", fontsize=14, fontweight="bold") plt.grid(axis="y", linestyle=":", alpha=0.6) for bar, score in zip(bars, scores): yval = bar.get_height() plt.text(bar.get_x() + bar.get_width() / 2.0, yval + 1.2, f"{score:.1f}%", ha="center", va="bottom", fontsize=11, fontweight="bold") plt.legend(loc="upper left") plt.tight_layout() plt.savefig(save_path, dpi=300) plt.close() print(f"Benchmark chart saved to {save_path}") # --------------------------------------------------------------------------- # Main Orchestrator # --------------------------------------------------------------------------- def main(): parser = argparse.ArgumentParser(description="Parvulus 2000-Trial Parallel Benchmark") parser.add_argument("--output_dir", type=str, default="./outputs/eval_parvulus_2000") parser.add_argument("--num_trials", type=int, default=50, help="Trials per task (default: 50 -> 2,000 total)") parser.add_argument("--num_gpus", type=int, default=4, help="Number of GPUs to use") parser.add_argument("--workers_per_gpu", type=int, default=3, help="Workers per GPU (default: 3 -> 12 workers)") parser.add_argument("--seed", type=int, default=42, help="Base random seed") parser.add_argument("--chunk_size", type=int, default=8) parser.add_argument("--replan_stride", type=int, default=8) parser.add_argument("--num_ode_steps", type=int, default=10) parser.add_argument("--resolution", type=int, default=256) parser.add_argument("--backend", type=str, default="parvulus", choices=["parvulus", "pure_cuda", "pytorch", "cuda_c", "cuda_graph"]) # Internal worker flag parser.add_argument("--worker_id", type=int, default=None) parser.add_argument("--gpu_id", type=int, default=None) args = parser.parse_args() # If launched as worker subprocess: if args.worker_id is not None: out_dir = Path(args.output_dir) queue_dir = out_dir / "queue" results_dir = out_dir / "results" tasks_meta_file = out_dir / "tasks_meta.json" run_worker( worker_id=args.worker_id, gpu_id=args.gpu_id, queue_dir=queue_dir, results_dir=results_dir, tasks_meta_file=tasks_meta_file, num_trials=args.num_trials, chunk_size=args.chunk_size, replan_stride=args.replan_stride, num_ode_steps=args.num_ode_steps, resolution=args.resolution, seed_offset=args.seed, backend=args.backend, ) return # Master orchestrator process out_dir = Path(args.output_dir) queue_dir = out_dir / "queue" results_dir = out_dir / "results" logs_dir = out_dir / "logs" out_dir.mkdir(parents=True, exist_ok=True) queue_dir.mkdir(parents=True, exist_ok=True) results_dir.mkdir(parents=True, exist_ok=True) logs_dir.mkdir(parents=True, exist_ok=True) # Build canonical tasks list if LIBERO_ROOT and LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path: sys.path.insert(0, str(LIBERO_ROOT)) from libero.libero import benchmark bench_dict = benchmark.get_benchmark_dict() tasks_meta = [] global_idx = 0 for suite_name in TASK_SUITES: suite = bench_dict[suite_name]() for task_id in range(suite.n_tasks): t = suite.get_task(task_id) tasks_meta.append({ "global_idx": global_idx, "suite_name": suite_name, "task_id": task_id, "task_name": t.name, "language": t.language, }) global_idx += 1 tasks_meta_file = out_dir / "tasks_meta.json" with open(tasks_meta_file, "w") as f: json.dump(tasks_meta, f, indent=2) total_tasks = len(tasks_meta) total_trials = total_tasks * args.num_trials total_workers = args.num_gpus * args.workers_per_gpu print("=" * 75) print(" PARVULUS 2,000-TRIAL BENCHMARK ORCHESTRATOR") print("=" * 75) print(f" Total Tasks: {total_tasks} across {len(TASK_SUITES)} canonical suites") print(f" Trials per Task: {args.num_trials}") print(f" Total Trials: {total_trials}") print(f" Hardware: {args.num_gpus} NVIDIA GPUs x {args.workers_per_gpu} workers = {total_workers} parallel workers") print(f" Output Directory: {out_dir}") print("=" * 75) # Check for already completed tasks completed = [i for i in range(total_tasks) if (results_dir / f"result_task_{i:02d}.json").exists()] if completed: print(f" [Resume] Found {len(completed)}/{total_tasks} already completed tasks. Skipping them.") # Spawn worker subprocesses workers = [] t_start = time.time() worker_id = 0 for gpu_id in range(args.num_gpus): for _ in range(args.workers_per_gpu): log_path = logs_dir / f"worker_{worker_id}_gpu_{gpu_id}.log" log_file = open(log_path, "w") cmd = [ sys.executable, __file__, "--output_dir", str(out_dir), "--num_trials", str(args.num_trials), "--chunk_size", str(args.chunk_size), "--replan_stride", str(args.replan_stride), "--num_ode_steps", str(args.num_ode_steps), "--resolution", str(args.resolution), "--backend", str(args.backend), "--seed", str(args.seed), "--worker_id", str(worker_id), "--gpu_id", str(gpu_id), ] proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT) workers.append({ "worker_id": worker_id, "gpu_id": gpu_id, "proc": proc, "log_file": log_file, }) worker_id += 1 print(f"\nSpawned {len(workers)} worker subprocesses across {args.num_gpus} GPUs. Monitoring execution...") # Monitoring loop last_print = 0 while True: time.sleep(5.0) # Check workers status all_finished = True for w in workers: ret = w["proc"].poll() if ret is None: all_finished = False # Scan results result_files = list(results_dir.glob("result_task_*.json")) num_done = len(result_files) now = time.time() if now - last_print >= 15.0 or num_done == total_tasks or all_finished: last_print = now elapsed = now - t_start # Calculate current stats total_succ = 0 trials_counted = 0 suite_counts: Dict[str, Dict[str, int]] = {s: {"succ": 0, "trials": 0} for s in TASK_SUITES} for rf in result_files: try: with open(rf) as f: rd = json.load(f) s = rd["suite_name"] succ = rd["successes"] tr = rd["num_trials"] total_succ += succ trials_counted += tr suite_counts[s]["succ"] += succ suite_counts[s]["trials"] += tr except Exception: pass cur_rate = (total_succ / max(1, trials_counted)) * 100 rate_per_sec = trials_counted / max(1e-5, elapsed) remaining_trials = total_trials - trials_counted eta_sec = remaining_trials / max(1e-5, rate_per_sec) if rate_per_sec > 0 else 0 suite_str = " | ".join([ f"{s.replace('libero_', '')[:4]}: {suite_counts[s]['succ']}/{suite_counts[s]['trials']}" for s in TASK_SUITES ]) print( f"[{time.strftime('%H:%M:%S')}] Tasks: {num_done:02d}/{total_tasks} | " f"Trials: {trials_counted:4d}/{total_trials} ({cur_rate:5.1f}% SR) | " f"Speed: {rate_per_sec:4.1f} tr/s | " f"Elapsed: {elapsed/60:4.1f}m | ETA: {eta_sec/60:4.1f}m | [{suite_str}]", flush=True, ) if num_done == total_tasks: print("\nAll 40 tasks have completed successfully!") break if all_finished and num_done < total_tasks: print(f"\n[Warning] All worker processes terminated but only {num_done}/{total_tasks} tasks completed.") break # Clean up file handles for w in workers: w["proc"].wait() w["log_file"].close() total_walltime = time.time() - t_start print(f"\nExecution finished in {total_walltime/60:.2f} minutes ({total_walltime:.1f} seconds).") # Aggregate full benchmark suite_summaries = {s: {"successes": 0, "trials": 0, "rate": 0.0} for s in TASK_SUITES} all_results = [] for idx in range(total_tasks): rf = results_dir / f"result_task_{idx:02d}.json" if not rf.exists(): continue with open(rf) as f: rd = json.load(f) s = rd["suite_name"] suite_summaries[s]["successes"] += rd["successes"] suite_summaries[s]["trials"] += rd["num_trials"] all_results.append(rd) for s in TASK_SUITES: trials = suite_summaries[s]["trials"] suite_summaries[s]["rate"] = (suite_summaries[s]["successes"] / trials) if trials > 0 else 0.0 total_succ = sum(suite_summaries[s]["successes"] for s in TASK_SUITES) total_eval_trials = sum(suite_summaries[s]["trials"] for s in TASK_SUITES) overall_rate = total_succ / max(1, total_eval_trials) print("\n" + "=" * 75) print(f" PARVULUS BENCHMARK RESULTS ({total_eval_trials} Total Trials across 40 Tasks)") print("=" * 75) for s in TASK_SUITES: r = suite_summaries[s] print(f" - {s:<18}: {r['successes']:4d} / {r['trials']:4d} ({r['rate'] * 100:6.2f}%)") if all(s in suite_summaries for s in ["libero_spatial", "libero_object", "libero_goal"]): s3_succ = sum(suite_summaries[s]["successes"] for s in ["libero_spatial", "libero_object", "libero_goal"]) s3_trials = sum(suite_summaries[s]["trials"] for s in ["libero_spatial", "libero_object", "libero_goal"]) s3_rate = s3_succ / max(1, s3_trials) print(" " + "-" * 60) print(f" - 3-Suite Subtotal : {s3_succ:4d} / {s3_trials:4d} ({s3_rate * 100:6.2f}%)") print(f" - OVERALL BENCHMARK : {total_succ:4d} / {total_eval_trials:4d} ({overall_rate * 100:6.2f}%)") print("=" * 75 + "\n") summary_file = out_dir / "benchmark_summary_2000.json" chart_file = out_dir / "benchmark_chart_2000.png" with open(summary_file, "w") as f: json.dump({ "overall_success_rate": overall_rate, "overall_successes": total_succ, "total_trials": total_eval_trials, "walltime_minutes": round(total_walltime / 60.0, 2), "trials_per_second": round(total_eval_trials / max(1e-5, total_walltime), 2), "suite_summaries": suite_summaries, "tasks": all_results, }, f, indent=2) plot_results( suite_summaries=suite_summaries, overall_rate=overall_rate, total_trials=total_eval_trials, save_path=str(chart_file), ) print(f"Summary JSON saved to {summary_file}") if __name__ == "__main__": main()