Download examples/simple/simple_eval.py from Tinghong-Ye22/code-Psi0-yth-v20: direct link, hf CLI and curl.
- Browser
- Download file 5.85 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v20/resolve/main/examples/simple/simple_eval.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v20/examples/simple/simple_eval.py
-
curl -L -o simple_eval.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v20/resolve/main/examples/simple/simple_eval.py
5.85 kB
| #!/usr/bin/env python3 | |
| from __future__ import annotations | |
| import argparse | |
| import os | |
| from pathlib import Path | |
| import socket | |
| import subprocess | |
| import sys | |
| import time | |
| from simple.evals.api import EvalConfig, EvalRunner | |
| REPO_ROOT = Path(__file__).resolve().parents[2] | |
| def _pick_free_port(start: int = 22085, end: int = 22999) -> int: | |
| for port in range(start, end + 1): | |
| with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: | |
| sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) | |
| if sock.connect_ex(("127.0.0.1", port)) != 0: | |
| return port | |
| raise RuntimeError(f"No free port found in range {start}-{end}") | |
| def _wait_for_port(host: str, port: int, tries: int, sleep_s: float) -> None: | |
| for _ in range(tries): | |
| with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: | |
| if sock.connect_ex((host, port)) == 0: | |
| return | |
| time.sleep(sleep_s) | |
| raise TimeoutError(f"Timed out waiting for {host}:{port}") | |
| def _build_server_cmd(args: argparse.Namespace, port: int) -> list[str]: | |
| cmd = [ | |
| sys.executable, | |
| "-m", | |
| "psi.deploy.psi0_serve_simple", | |
| "--host", | |
| args.server_host, | |
| "--port", | |
| str(port), | |
| "--policy", | |
| args.policy, | |
| "--run-dir", | |
| args.run_dir, | |
| "--ckpt-step", | |
| str(args.ckpt_step), | |
| "--device", | |
| args.device, | |
| ] | |
| if args.action_exec_horizon is not None: | |
| cmd.extend(["--action-exec-horizon", str(args.action_exec_horizon)]) | |
| if args.rtc: | |
| cmd.append("--rtc") | |
| return cmd | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description="Evaluate a PSI checkpoint with SIMPLE.") | |
| parser.add_argument("--run-dir", required=True, help="PSI run directory.") | |
| parser.add_argument("--ckpt-step", required=True, help="Checkpoint step or 'latest'.") | |
| parser.add_argument("--env-id", default="simple/G1WholebodyBendPick-v0") | |
| parser.add_argument("--policy", default="psi0") | |
| parser.add_argument("--data-dir", required=True, help="LeRobot dataset root for SIMPLE eval.") | |
| parser.add_argument("--split", default="train") | |
| parser.add_argument("--device", default="cuda:0") | |
| parser.add_argument("--server-host", default="0.0.0.0") | |
| parser.add_argument("--host", default="localhost", help="Host used by SIMPLE eval client.") | |
| parser.add_argument("--port", type=int, help="Policy server port. Defaults to a free port.") | |
| parser.add_argument("--sim-mode", default="mujoco_isaac") | |
| parser.add_argument("--data-format", default="lerobot") | |
| parser.add_argument("--eval-dir", default="data/evals") | |
| parser.add_argument("--num-episodes", type=int, default=10) | |
| parser.add_argument("--episode-start", type=int, default=0) | |
| parser.add_argument("--num-workers", type=int, default=1) | |
| parser.add_argument("--max-episode-steps", type=int, default=360) | |
| parser.add_argument("--success-criteria", type=float, default=0.9) | |
| parser.add_argument("--action-exec-horizon", type=int) | |
| parser.add_argument("--wait-tries", type=int, default=1200) | |
| parser.add_argument("--wait-sleep-s", type=float, default=0.1) | |
| parser.add_argument("--server-log", help="Optional path for server stdout/stderr.") | |
| parser.add_argument("--rtc", action="store_true") | |
| parser.add_argument("--headless", action="store_true", default=True) | |
| parser.add_argument("--no-headless", dest="headless", action="store_false") | |
| parser.add_argument("--save-video", action="store_true", default=True) | |
| parser.add_argument("--no-save-video", dest="save_video", action="store_false") | |
| parser.add_argument("--dry-run", action="store_true") | |
| args = parser.parse_args() | |
| port = args.port or _pick_free_port() | |
| server_cmd = _build_server_cmd(args, port) | |
| eval_config = EvalConfig( | |
| env_id=args.env_id, | |
| policy=args.policy, | |
| split=args.split, | |
| host=args.host, | |
| port=port, | |
| data_format=args.data_format, | |
| sim_mode=args.sim_mode, | |
| headless=args.headless, | |
| eval_dir=args.eval_dir, | |
| max_episode_steps=args.max_episode_steps, | |
| num_episodes=args.num_episodes, | |
| episode_start=args.episode_start, | |
| data_dir=args.data_dir, | |
| success_criteria=args.success_criteria, | |
| save_video=args.save_video, | |
| num_workers=args.num_workers, | |
| ) | |
| print(f"server_cmd={' '.join(server_cmd)}") | |
| print(f"eval_config={eval_config}") | |
| if args.dry_run: | |
| return 0 | |
| env = os.environ.copy() | |
| env["PYTHONPATH"] = f"{REPO_ROOT / 'src'}:{env.get('PYTHONPATH', '')}".rstrip(":") | |
| log_handle = None | |
| server = None | |
| try: | |
| if args.server_log: | |
| log_path = Path(args.server_log) | |
| log_path.parent.mkdir(parents=True, exist_ok=True) | |
| log_handle = log_path.open("w") | |
| server = subprocess.Popen( | |
| server_cmd, | |
| cwd=REPO_ROOT, | |
| env=env, | |
| stdout=log_handle, | |
| stderr=subprocess.STDOUT, | |
| ) | |
| else: | |
| server = subprocess.Popen(server_cmd, cwd=REPO_ROOT, env=env) | |
| _wait_for_port(args.host, port, args.wait_tries, args.wait_sleep_s) | |
| result = EvalRunner(eval_config).run() | |
| print(f"success_rate={result.success_rate:.6f}") | |
| print(f"log_path={result.log_path}") | |
| print(f"eval_dir={result.eval_dir}") | |
| return 0 | |
| finally: | |
| if server is not None: | |
| server.terminate() | |
| try: | |
| server.wait(timeout=10) | |
| except subprocess.TimeoutExpired: | |
| server.kill() | |
| server.wait(timeout=5) | |
| if log_handle is not None: | |
| log_handle.close() | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |