File size: 5,848 Bytes
6eb47b8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | #!/usr/bin/env python3
from __future__ import annotations
import argparse
import os
from pathlib import Path
import socket
import subprocess
import sys
import time
from simple.evals.api import EvalConfig, EvalRunner
REPO_ROOT = Path(__file__).resolve().parents[2]
def _pick_free_port(start: int = 22085, end: int = 22999) -> int:
for port in range(start, end + 1):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if sock.connect_ex(("127.0.0.1", port)) != 0:
return port
raise RuntimeError(f"No free port found in range {start}-{end}")
def _wait_for_port(host: str, port: int, tries: int, sleep_s: float) -> None:
for _ in range(tries):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
if sock.connect_ex((host, port)) == 0:
return
time.sleep(sleep_s)
raise TimeoutError(f"Timed out waiting for {host}:{port}")
def _build_server_cmd(args: argparse.Namespace, port: int) -> list[str]:
cmd = [
sys.executable,
"-m",
"psi.deploy.psi0_serve_simple",
"--host",
args.server_host,
"--port",
str(port),
"--policy",
args.policy,
"--run-dir",
args.run_dir,
"--ckpt-step",
str(args.ckpt_step),
"--device",
args.device,
]
if args.action_exec_horizon is not None:
cmd.extend(["--action-exec-horizon", str(args.action_exec_horizon)])
if args.rtc:
cmd.append("--rtc")
return cmd
def main() -> int:
parser = argparse.ArgumentParser(description="Evaluate a PSI checkpoint with SIMPLE.")
parser.add_argument("--run-dir", required=True, help="PSI run directory.")
parser.add_argument("--ckpt-step", required=True, help="Checkpoint step or 'latest'.")
parser.add_argument("--env-id", default="simple/G1WholebodyBendPick-v0")
parser.add_argument("--policy", default="psi0")
parser.add_argument("--data-dir", required=True, help="LeRobot dataset root for SIMPLE eval.")
parser.add_argument("--split", default="train")
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--server-host", default="0.0.0.0")
parser.add_argument("--host", default="localhost", help="Host used by SIMPLE eval client.")
parser.add_argument("--port", type=int, help="Policy server port. Defaults to a free port.")
parser.add_argument("--sim-mode", default="mujoco_isaac")
parser.add_argument("--data-format", default="lerobot")
parser.add_argument("--eval-dir", default="data/evals")
parser.add_argument("--num-episodes", type=int, default=10)
parser.add_argument("--episode-start", type=int, default=0)
parser.add_argument("--num-workers", type=int, default=1)
parser.add_argument("--max-episode-steps", type=int, default=360)
parser.add_argument("--success-criteria", type=float, default=0.9)
parser.add_argument("--action-exec-horizon", type=int)
parser.add_argument("--wait-tries", type=int, default=1200)
parser.add_argument("--wait-sleep-s", type=float, default=0.1)
parser.add_argument("--server-log", help="Optional path for server stdout/stderr.")
parser.add_argument("--rtc", action="store_true")
parser.add_argument("--headless", action="store_true", default=True)
parser.add_argument("--no-headless", dest="headless", action="store_false")
parser.add_argument("--save-video", action="store_true", default=True)
parser.add_argument("--no-save-video", dest="save_video", action="store_false")
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
port = args.port or _pick_free_port()
server_cmd = _build_server_cmd(args, port)
eval_config = EvalConfig(
env_id=args.env_id,
policy=args.policy,
split=args.split,
host=args.host,
port=port,
data_format=args.data_format,
sim_mode=args.sim_mode,
headless=args.headless,
eval_dir=args.eval_dir,
max_episode_steps=args.max_episode_steps,
num_episodes=args.num_episodes,
episode_start=args.episode_start,
data_dir=args.data_dir,
success_criteria=args.success_criteria,
save_video=args.save_video,
num_workers=args.num_workers,
)
print(f"server_cmd={' '.join(server_cmd)}")
print(f"eval_config={eval_config}")
if args.dry_run:
return 0
env = os.environ.copy()
env["PYTHONPATH"] = f"{REPO_ROOT / 'src'}:{env.get('PYTHONPATH', '')}".rstrip(":")
log_handle = None
server = None
try:
if args.server_log:
log_path = Path(args.server_log)
log_path.parent.mkdir(parents=True, exist_ok=True)
log_handle = log_path.open("w")
server = subprocess.Popen(
server_cmd,
cwd=REPO_ROOT,
env=env,
stdout=log_handle,
stderr=subprocess.STDOUT,
)
else:
server = subprocess.Popen(server_cmd, cwd=REPO_ROOT, env=env)
_wait_for_port(args.host, port, args.wait_tries, args.wait_sleep_s)
result = EvalRunner(eval_config).run()
print(f"success_rate={result.success_rate:.6f}")
print(f"log_path={result.log_path}")
print(f"eval_dir={result.eval_dir}")
return 0
finally:
if server is not None:
server.terminate()
try:
server.wait(timeout=10)
except subprocess.TimeoutExpired:
server.kill()
server.wait(timeout=5)
if log_handle is not None:
log_handle.close()
if __name__ == "__main__":
raise SystemExit(main())
|