Download eval_benchmark_parallel.py from aasivas/Parvulus: direct link, hf CLI and curl.
- Browser
- Download file 21.5 kB
-
https://huggingface.co/aasivas/Parvulus/resolve/main/eval_benchmark_parallel.py
- Command line
-
hf download hf://aasivas/Parvulus/eval_benchmark_parallel.py
-
curl -L -o eval_benchmark_parallel.py https://huggingface.co/aasivas/Parvulus/resolve/main/eval_benchmark_parallel.py
21.5 kB
| #!/usr/bin/env python3 | |
| """High-Performance Parallel Benchmark Evaluator for Parvulus across LIBERO. | |
| Runs all 40 canonical tasks across 4 suites (50 trials each = 2,000 trials) | |
| distributed dynamically across multiple GPUs and worker processes. | |
| Uses atomic lock queue for dynamic load balancing and fault tolerance. | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| from pathlib import Path | |
| from typing import Any, Dict, List, Optional, Tuple | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| TASK_SUITES = ["libero_spatial", "libero_object", "libero_goal", "libero_10"] | |
| MAX_STEPS = { | |
| "libero_spatial": 500, | |
| "libero_object": 500, | |
| "libero_goal": 500, | |
| "libero_10": 600, | |
| } | |
| SCRIPT_DIR = Path(__file__).resolve().parent | |
| PROJECT_ROOT = SCRIPT_DIR | |
| LIBERO_ROOT = Path(os.environ["LIBERO_ROOT"]) if "LIBERO_ROOT" in os.environ else None | |
| # --------------------------------------------------------------------------- | |
| # Worker process entrypoint | |
| # --------------------------------------------------------------------------- | |
| def run_worker( | |
| worker_id: int, | |
| gpu_id: int, | |
| queue_dir: Path, | |
| results_dir: Path, | |
| tasks_meta_file: Path, | |
| num_trials: int, | |
| chunk_size: int, | |
| replan_stride: int, | |
| num_ode_steps: int, | |
| resolution: int, | |
| seed_offset: int, | |
| backend: str = "cuda_c", | |
| ): | |
| """Worker process: loops over dynamic task queue and executes rollouts.""" | |
| # Ensure correct environment variables before importing heavy frameworks | |
| os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu_id) | |
| os.environ["MUJOCO_GL"] = "egl" | |
| os.environ["TOKENIZERS_PARALLELISM"] = "false" | |
| if str(SCRIPT_DIR) not in sys.path: | |
| sys.path.insert(0, str(SCRIPT_DIR)) | |
| if LIBERO_ROOT and LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path: | |
| sys.path.insert(0, str(LIBERO_ROOT)) | |
| import torch | |
| from infer import ParvulusInference | |
| from libero.libero import benchmark, get_libero_path | |
| from libero.libero.envs import OffScreenRenderEnv | |
| # Load tasks metadata | |
| with open(tasks_meta_file, "r") as f: | |
| tasks = json.load(f) | |
| device = "cuda:0" | |
| print(f"[Worker {worker_id} (GPU {gpu_id})] Initializing Parvulus policy on {device} (backend={backend})...", flush=True) | |
| infer = ParvulusInference.from_pretrained( | |
| pretrained_path=str(SCRIPT_DIR), | |
| device=device, | |
| backend=backend, | |
| use_kv_cache=True, | |
| num_ode_steps=num_ode_steps, | |
| chunk_size=chunk_size, | |
| replan_stride=replan_stride, | |
| resolution=resolution, | |
| ) | |
| bench_dict = benchmark.get_benchmark_dict() | |
| loaded_suites: Dict[str, Any] = {} | |
| settle_noop = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0]) | |
| retreat_action = np.array([0.0, 0.0, 0.6, 0.0, 0.0, 0.0, -1.0]) | |
| print(f"[Worker {worker_id} (GPU {gpu_id})] Ready. Polling task queue...", flush=True) | |
| while True: | |
| # Atomic claim of next unclaimed task | |
| claimed_idx = None | |
| for idx in range(len(tasks)): | |
| result_file = results_dir / f"result_task_{idx:02d}.json" | |
| if result_file.exists(): | |
| continue | |
| lock_file = queue_dir / f"task_{idx:02d}.lock" | |
| try: | |
| fd = os.open(lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY) | |
| with os.fdopen(fd, "w") as lf: | |
| lf.write(f"worker_{worker_id}_pid_{os.getpid()}\n") | |
| claimed_idx = idx | |
| break | |
| except FileExistsError: | |
| continue | |
| if claimed_idx is None: | |
| # Check if all tasks have results | |
| all_done = all((results_dir / f"result_task_{i:02d}.json").exists() for i in range(len(tasks))) | |
| if all_done: | |
| break | |
| time.sleep(1.0) | |
| continue | |
| task_info = tasks[claimed_idx] | |
| suite_name = task_info["suite_name"] | |
| task_id = task_info["task_id"] | |
| task_desc = task_info["language"] | |
| task_name = task_info["task_name"] | |
| max_steps = MAX_STEPS.get(suite_name, 500) | |
| print( | |
| f"[Worker {worker_id} (GPU {gpu_id})] Claimed Task {claimed_idx:02d} " | |
| f"[{suite_name} #{task_id}]: '{task_desc[:40]}...'", | |
| flush=True, | |
| ) | |
| if suite_name not in loaded_suites: | |
| loaded_suites[suite_name] = bench_dict[suite_name]() | |
| suite = loaded_suites[suite_name] | |
| task_obj = suite.get_task(task_id) | |
| # Load canonical initial states | |
| with torch.serialization.safe_globals([np.core.multiarray._reconstruct, np.ndarray, np.dtype, np.dtypes.Float64DType]): | |
| init_states = suite.get_task_init_states(task_id) | |
| bddl_file = Path(get_libero_path("bddl_files")) / task_obj.problem_folder / task_obj.bddl_file | |
| env = OffScreenRenderEnv( | |
| bddl_file_name=str(bddl_file), | |
| camera_heights=resolution, | |
| camera_widths=resolution, | |
| ) | |
| infer.reset(task_description=task_desc) | |
| trial_records = [] | |
| task_successes = 0 | |
| task_t0 = time.time() | |
| for trial in range(num_trials): | |
| trial_seed = seed_offset + claimed_idx * 1000 + trial | |
| env.seed(trial_seed) | |
| torch.manual_seed(trial_seed) | |
| np.random.seed(trial_seed) | |
| obs = env.reset() | |
| init_state = init_states[trial % len(init_states)] | |
| obs = env.set_init_state(init_state) | |
| infer.reset(task_description=task_desc) | |
| # Settle | |
| for _ in range(5): | |
| obs, _, _, _ = env.step(settle_noop) | |
| success = False | |
| step_count = 0 | |
| trial_t0 = time.time() | |
| while step_count < max_steps and not success: | |
| agent_img = obs["agentview_image"] | |
| wrist_img = obs["robot0_eye_in_hand_image"] | |
| action = infer.step(agent_img, wrist_img) | |
| obs, reward, done, info = env.step(action) | |
| step_count += 1 | |
| if done: | |
| # Probe | |
| for _ in range(10): | |
| obs, _, d, _ = env.step(action) | |
| if d: | |
| done = True | |
| # Retreat | |
| for _ in range(15): | |
| obs, _, d, _ = env.step(retreat_action) | |
| # Settle | |
| for _ in range(5): | |
| obs, _, d, _ = env.step(np.zeros(7)) | |
| success = bool(done) | |
| break | |
| trial_elapsed = time.time() - trial_t0 | |
| if success: | |
| task_successes += 1 | |
| trial_records.append({ | |
| "trial": trial, | |
| "success": success, | |
| "steps": step_count, | |
| "elapsed": round(trial_elapsed, 2), | |
| }) | |
| if (trial + 1) % 10 == 0 or trial == num_trials - 1: | |
| cur_rate = task_successes / (trial + 1) * 100 | |
| print( | |
| f" [Worker {worker_id} | Task {claimed_idx:02d}] " | |
| f"Trial {trial+1:02d}/{num_trials}: {task_successes}/{trial+1} ({cur_rate:.1f}%) " | |
| f"[{trial_elapsed:.2f}s, {step_count} steps]", | |
| flush=True, | |
| ) | |
| env.close() | |
| task_elapsed = time.time() - task_t0 | |
| # Save task result atomically | |
| result_payload = { | |
| "global_task_idx": claimed_idx, | |
| "suite_name": suite_name, | |
| "task_id": task_id, | |
| "task_name": task_name, | |
| "task_desc": task_desc, | |
| "num_trials": num_trials, | |
| "successes": task_successes, | |
| "success_rate": task_successes / num_trials, | |
| "task_elapsed_seconds": round(task_elapsed, 2), | |
| "worker_id": worker_id, | |
| "gpu_id": gpu_id, | |
| "trials": trial_records, | |
| } | |
| tmp_result = results_dir / f"result_task_{claimed_idx:02d}.tmp" | |
| final_result = results_dir / f"result_task_{claimed_idx:02d}.json" | |
| with open(tmp_result, "w") as f: | |
| json.dump(result_payload, f, indent=2) | |
| os.replace(tmp_result, final_result) | |
| print( | |
| f"[Worker {worker_id} (GPU {gpu_id})] COMPLETED Task {claimed_idx:02d} " | |
| f"({suite_name} #{task_id}) -> {task_successes}/{num_trials} " | |
| f"({task_successes/num_trials*100:.1f}%) in {task_elapsed:.1f}s", | |
| flush=True, | |
| ) | |
| print(f"[Worker {worker_id} (GPU {gpu_id})] Exiting cleanly.", flush=True) | |
| # --------------------------------------------------------------------------- | |
| # Plotting helper | |
| # --------------------------------------------------------------------------- | |
| def plot_results(suite_summaries: Dict[str, Dict], overall_rate: float, total_trials: int, save_path: str): | |
| suites = list(suite_summaries.keys()) | |
| labels = [s.replace("libero_", "").upper() for s in suites] | |
| scores = [suite_summaries[s]["rate"] * 100 for s in suites] | |
| if all(s in suite_summaries for s in ["libero_spatial", "libero_object", "libero_goal"]): | |
| s3_succ = sum(suite_summaries[s]["successes"] for s in ["libero_spatial", "libero_object", "libero_goal"]) | |
| s3_trials = sum(suite_summaries[s]["trials"] for s in ["libero_spatial", "libero_object", "libero_goal"]) | |
| labels.append("3-SUITE") | |
| scores.append((s3_succ / s3_trials) * 100) | |
| labels.append("OVERALL") | |
| scores.append(overall_rate * 100) | |
| colors = ["#4C72B0", "#55A868", "#C44E52", "#8172B3", "#CCB974", "#64B5CD"][:len(labels)] | |
| plt.figure(figsize=(max(8, len(labels) * 1.5), 5.5)) | |
| bars = plt.bar(labels, scores, color=colors, width=0.55, edgecolor="black", linewidth=1.2) | |
| plt.axhline(90, color="red", linestyle="--", linewidth=1.5, label="Gold Reference (90%)") | |
| plt.ylim(0, 105) | |
| plt.ylabel("Success Rate (%)", fontsize=12, fontweight="bold") | |
| plt.title(f"PARVULUS LIBERO Benchmark Evaluation ({total_trials} Trials)", fontsize=14, fontweight="bold") | |
| plt.grid(axis="y", linestyle=":", alpha=0.6) | |
| for bar, score in zip(bars, scores): | |
| yval = bar.get_height() | |
| plt.text(bar.get_x() + bar.get_width() / 2.0, yval + 1.2, f"{score:.1f}%", ha="center", va="bottom", fontsize=11, fontweight="bold") | |
| plt.legend(loc="upper left") | |
| plt.tight_layout() | |
| plt.savefig(save_path, dpi=300) | |
| plt.close() | |
| print(f"Benchmark chart saved to {save_path}") | |
| # --------------------------------------------------------------------------- | |
| # Main Orchestrator | |
| # --------------------------------------------------------------------------- | |
| def main(): | |
| parser = argparse.ArgumentParser(description="Parvulus 2000-Trial Parallel Benchmark") | |
| parser.add_argument("--output_dir", type=str, default="./outputs/eval_parvulus_2000") | |
| parser.add_argument("--num_trials", type=int, default=50, help="Trials per task (default: 50 -> 2,000 total)") | |
| parser.add_argument("--num_gpus", type=int, default=4, help="Number of GPUs to use") | |
| parser.add_argument("--workers_per_gpu", type=int, default=3, help="Workers per GPU (default: 3 -> 12 workers)") | |
| parser.add_argument("--seed", type=int, default=42, help="Base random seed") | |
| parser.add_argument("--chunk_size", type=int, default=8) | |
| parser.add_argument("--replan_stride", type=int, default=8) | |
| parser.add_argument("--num_ode_steps", type=int, default=10) | |
| parser.add_argument("--resolution", type=int, default=256) | |
| parser.add_argument("--backend", type=str, default="parvulus", choices=["parvulus", "pure_cuda", "pytorch", "cuda_c", "cuda_graph"]) | |
| # Internal worker flag | |
| parser.add_argument("--worker_id", type=int, default=None) | |
| parser.add_argument("--gpu_id", type=int, default=None) | |
| args = parser.parse_args() | |
| # If launched as worker subprocess: | |
| if args.worker_id is not None: | |
| out_dir = Path(args.output_dir) | |
| queue_dir = out_dir / "queue" | |
| results_dir = out_dir / "results" | |
| tasks_meta_file = out_dir / "tasks_meta.json" | |
| run_worker( | |
| worker_id=args.worker_id, | |
| gpu_id=args.gpu_id, | |
| queue_dir=queue_dir, | |
| results_dir=results_dir, | |
| tasks_meta_file=tasks_meta_file, | |
| num_trials=args.num_trials, | |
| chunk_size=args.chunk_size, | |
| replan_stride=args.replan_stride, | |
| num_ode_steps=args.num_ode_steps, | |
| resolution=args.resolution, | |
| seed_offset=args.seed, | |
| backend=args.backend, | |
| ) | |
| return | |
| # Master orchestrator process | |
| out_dir = Path(args.output_dir) | |
| queue_dir = out_dir / "queue" | |
| results_dir = out_dir / "results" | |
| logs_dir = out_dir / "logs" | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| queue_dir.mkdir(parents=True, exist_ok=True) | |
| results_dir.mkdir(parents=True, exist_ok=True) | |
| logs_dir.mkdir(parents=True, exist_ok=True) | |
| # Build canonical tasks list | |
| if LIBERO_ROOT and LIBERO_ROOT.exists() and str(LIBERO_ROOT) not in sys.path: | |
| sys.path.insert(0, str(LIBERO_ROOT)) | |
| from libero.libero import benchmark | |
| bench_dict = benchmark.get_benchmark_dict() | |
| tasks_meta = [] | |
| global_idx = 0 | |
| for suite_name in TASK_SUITES: | |
| suite = bench_dict[suite_name]() | |
| for task_id in range(suite.n_tasks): | |
| t = suite.get_task(task_id) | |
| tasks_meta.append({ | |
| "global_idx": global_idx, | |
| "suite_name": suite_name, | |
| "task_id": task_id, | |
| "task_name": t.name, | |
| "language": t.language, | |
| }) | |
| global_idx += 1 | |
| tasks_meta_file = out_dir / "tasks_meta.json" | |
| with open(tasks_meta_file, "w") as f: | |
| json.dump(tasks_meta, f, indent=2) | |
| total_tasks = len(tasks_meta) | |
| total_trials = total_tasks * args.num_trials | |
| total_workers = args.num_gpus * args.workers_per_gpu | |
| print("=" * 75) | |
| print(" PARVULUS 2,000-TRIAL BENCHMARK ORCHESTRATOR") | |
| print("=" * 75) | |
| print(f" Total Tasks: {total_tasks} across {len(TASK_SUITES)} canonical suites") | |
| print(f" Trials per Task: {args.num_trials}") | |
| print(f" Total Trials: {total_trials}") | |
| print(f" Hardware: {args.num_gpus} NVIDIA GPUs x {args.workers_per_gpu} workers = {total_workers} parallel workers") | |
| print(f" Output Directory: {out_dir}") | |
| print("=" * 75) | |
| # Check for already completed tasks | |
| completed = [i for i in range(total_tasks) if (results_dir / f"result_task_{i:02d}.json").exists()] | |
| if completed: | |
| print(f" [Resume] Found {len(completed)}/{total_tasks} already completed tasks. Skipping them.") | |
| # Spawn worker subprocesses | |
| workers = [] | |
| t_start = time.time() | |
| worker_id = 0 | |
| for gpu_id in range(args.num_gpus): | |
| for _ in range(args.workers_per_gpu): | |
| log_path = logs_dir / f"worker_{worker_id}_gpu_{gpu_id}.log" | |
| log_file = open(log_path, "w") | |
| cmd = [ | |
| sys.executable, | |
| __file__, | |
| "--output_dir", str(out_dir), | |
| "--num_trials", str(args.num_trials), | |
| "--chunk_size", str(args.chunk_size), | |
| "--replan_stride", str(args.replan_stride), | |
| "--num_ode_steps", str(args.num_ode_steps), | |
| "--resolution", str(args.resolution), | |
| "--backend", str(args.backend), | |
| "--seed", str(args.seed), | |
| "--worker_id", str(worker_id), | |
| "--gpu_id", str(gpu_id), | |
| ] | |
| proc = subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT) | |
| workers.append({ | |
| "worker_id": worker_id, | |
| "gpu_id": gpu_id, | |
| "proc": proc, | |
| "log_file": log_file, | |
| }) | |
| worker_id += 1 | |
| print(f"\nSpawned {len(workers)} worker subprocesses across {args.num_gpus} GPUs. Monitoring execution...") | |
| # Monitoring loop | |
| last_print = 0 | |
| while True: | |
| time.sleep(5.0) | |
| # Check workers status | |
| all_finished = True | |
| for w in workers: | |
| ret = w["proc"].poll() | |
| if ret is None: | |
| all_finished = False | |
| # Scan results | |
| result_files = list(results_dir.glob("result_task_*.json")) | |
| num_done = len(result_files) | |
| now = time.time() | |
| if now - last_print >= 15.0 or num_done == total_tasks or all_finished: | |
| last_print = now | |
| elapsed = now - t_start | |
| # Calculate current stats | |
| total_succ = 0 | |
| trials_counted = 0 | |
| suite_counts: Dict[str, Dict[str, int]] = {s: {"succ": 0, "trials": 0} for s in TASK_SUITES} | |
| for rf in result_files: | |
| try: | |
| with open(rf) as f: | |
| rd = json.load(f) | |
| s = rd["suite_name"] | |
| succ = rd["successes"] | |
| tr = rd["num_trials"] | |
| total_succ += succ | |
| trials_counted += tr | |
| suite_counts[s]["succ"] += succ | |
| suite_counts[s]["trials"] += tr | |
| except Exception: | |
| pass | |
| cur_rate = (total_succ / max(1, trials_counted)) * 100 | |
| rate_per_sec = trials_counted / max(1e-5, elapsed) | |
| remaining_trials = total_trials - trials_counted | |
| eta_sec = remaining_trials / max(1e-5, rate_per_sec) if rate_per_sec > 0 else 0 | |
| suite_str = " | ".join([ | |
| f"{s.replace('libero_', '')[:4]}: {suite_counts[s]['succ']}/{suite_counts[s]['trials']}" | |
| for s in TASK_SUITES | |
| ]) | |
| print( | |
| f"[{time.strftime('%H:%M:%S')}] Tasks: {num_done:02d}/{total_tasks} | " | |
| f"Trials: {trials_counted:4d}/{total_trials} ({cur_rate:5.1f}% SR) | " | |
| f"Speed: {rate_per_sec:4.1f} tr/s | " | |
| f"Elapsed: {elapsed/60:4.1f}m | ETA: {eta_sec/60:4.1f}m | [{suite_str}]", | |
| flush=True, | |
| ) | |
| if num_done == total_tasks: | |
| print("\nAll 40 tasks have completed successfully!") | |
| break | |
| if all_finished and num_done < total_tasks: | |
| print(f"\n[Warning] All worker processes terminated but only {num_done}/{total_tasks} tasks completed.") | |
| break | |
| # Clean up file handles | |
| for w in workers: | |
| w["proc"].wait() | |
| w["log_file"].close() | |
| total_walltime = time.time() - t_start | |
| print(f"\nExecution finished in {total_walltime/60:.2f} minutes ({total_walltime:.1f} seconds).") | |
| # Aggregate full benchmark | |
| suite_summaries = {s: {"successes": 0, "trials": 0, "rate": 0.0} for s in TASK_SUITES} | |
| all_results = [] | |
| for idx in range(total_tasks): | |
| rf = results_dir / f"result_task_{idx:02d}.json" | |
| if not rf.exists(): | |
| continue | |
| with open(rf) as f: | |
| rd = json.load(f) | |
| s = rd["suite_name"] | |
| suite_summaries[s]["successes"] += rd["successes"] | |
| suite_summaries[s]["trials"] += rd["num_trials"] | |
| all_results.append(rd) | |
| for s in TASK_SUITES: | |
| trials = suite_summaries[s]["trials"] | |
| suite_summaries[s]["rate"] = (suite_summaries[s]["successes"] / trials) if trials > 0 else 0.0 | |
| total_succ = sum(suite_summaries[s]["successes"] for s in TASK_SUITES) | |
| total_eval_trials = sum(suite_summaries[s]["trials"] for s in TASK_SUITES) | |
| overall_rate = total_succ / max(1, total_eval_trials) | |
| print("\n" + "=" * 75) | |
| print(f" PARVULUS BENCHMARK RESULTS ({total_eval_trials} Total Trials across 40 Tasks)") | |
| print("=" * 75) | |
| for s in TASK_SUITES: | |
| r = suite_summaries[s] | |
| print(f" - {s:<18}: {r['successes']:4d} / {r['trials']:4d} ({r['rate'] * 100:6.2f}%)") | |
| if all(s in suite_summaries for s in ["libero_spatial", "libero_object", "libero_goal"]): | |
| s3_succ = sum(suite_summaries[s]["successes"] for s in ["libero_spatial", "libero_object", "libero_goal"]) | |
| s3_trials = sum(suite_summaries[s]["trials"] for s in ["libero_spatial", "libero_object", "libero_goal"]) | |
| s3_rate = s3_succ / max(1, s3_trials) | |
| print(" " + "-" * 60) | |
| print(f" - 3-Suite Subtotal : {s3_succ:4d} / {s3_trials:4d} ({s3_rate * 100:6.2f}%)") | |
| print(f" - OVERALL BENCHMARK : {total_succ:4d} / {total_eval_trials:4d} ({overall_rate * 100:6.2f}%)") | |
| print("=" * 75 + "\n") | |
| summary_file = out_dir / "benchmark_summary_2000.json" | |
| chart_file = out_dir / "benchmark_chart_2000.png" | |
| with open(summary_file, "w") as f: | |
| json.dump({ | |
| "overall_success_rate": overall_rate, | |
| "overall_successes": total_succ, | |
| "total_trials": total_eval_trials, | |
| "walltime_minutes": round(total_walltime / 60.0, 2), | |
| "trials_per_second": round(total_eval_trials / max(1e-5, total_walltime), 2), | |
| "suite_summaries": suite_summaries, | |
| "tasks": all_results, | |
| }, f, indent=2) | |
| plot_results( | |
| suite_summaries=suite_summaries, | |
| overall_rate=overall_rate, | |
| total_trials=total_eval_trials, | |
| save_path=str(chart_file), | |
| ) | |
| print(f"Summary JSON saved to {summary_file}") | |
| if __name__ == "__main__": | |
| main() | |