#!/usr/bin/env bash set -euo pipefail if [[ $# -ne 7 ]]; then echo "usage: $0 " >&2 exit 64 fi run_dir=$1 step=$2 gpu=$3 port=$4 policy_seed=$5 arm=$6 eval_root=$7 project="${OPTR_PROJECT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" runtime="${OPTR_RUNTIME_ROOT:?set OPTR_RUNTIME_ROOT to the runtime environment root}" simpler_root="${SIMPLER_ROOT:-${runtime}/SimplerEnv}" episodes_per_task=24 output_root="${eval_root}/policy_seed_${policy_seed}" step_dir="${output_root}/step_${step}" server_pid="" mkdir -p "${step_dir}" trap 'code=$?; trap - EXIT; if [[ -n "${server_pid}" ]]; then kill "${server_pid}" 2>/dev/null || true; wait "${server_pid}" 2>/dev/null || true; fi; printf "%s\n" "${code}" >"${step_dir}/exitcode.tmp.$$"; mv -f "${step_dir}/exitcode.tmp.$$" "${step_dir}/exitcode"; exit "${code}"' EXIT resolve_checkpoint() { "${runtime}/.venv/bin/python" - "${run_dir}" "${step}" <<'PY' import json import os import sys from pathlib import Path run_dir = Path(sys.argv[1]) step = int(sys.argv[2]) checkpoint_dir = run_dir / "checkpoints" / f"steps_{step}_accelerate" commit_path = checkpoint_dir / "checkpoint.json" commit = json.loads(commit_path.read_text(encoding="utf-8")) if commit["completed_steps"] != step or commit["world_size"] != 2: raise RuntimeError(f"checkpoint identity mismatch: {checkpoint_dir}") for entry in commit["files"]: path = checkpoint_dir / entry["name"] if not path.is_file() or path.stat().st_size != entry["bytes"]: raise RuntimeError(f"incomplete checkpoint file: {path}") source_record = next(entry for entry in commit["files"] if entry["name"] == "pytorch_model.pt") source = checkpoint_dir / source_record["name"] destination = run_dir / "checkpoints" / f"steps_{step}_pytorch_model.pt" try: os.link(source, destination) except FileExistsError: # Parallel policy seeds share this verified compatibility link. pass metadata = { "schema_version": 1, "completed_steps": step, "world_size": 2, "bytes": destination.stat().st_size, } sidecar = Path(f"{destination}.json") temporary = Path(f"{sidecar}.tmp.{os.getpid()}") temporary.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(temporary, sidecar) print(destination) PY } validate_task_summary() { local summary=$1 local task=$2 "${runtime}/.venv/bin/python" - "${summary}" "${step}" "${task}" "${policy_seed}" "${episodes_per_task}" <<'PY' import json import sys from pathlib import Path summary = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) step = int(sys.argv[2]) task = sys.argv[3] seed = int(sys.argv[4]) episodes = int(sys.argv[5]) if ( summary.get("status") != "complete_no_video_strict_success_panel" or summary.get("checkpoint_step") != step or summary.get("task") != task or summary.get("policy_seeds") != [seed] or summary.get("rollout_count") != episodes ): raise RuntimeError(f"invalid task summary: {sys.argv[1]}") PY } run_panel() { local checkpoint=$1 local task=$2 local panel_dir="${step_dir}/${task}" local seed_dir="${panel_dir}/seed_${policy_seed}" if [[ -f "${panel_dir}/summary.json" ]]; then validate_task_summary "${panel_dir}/summary.json" "${task}" return fi case "${task}" in spoon) env_name=PutSpoonOnTableClothInScene-v0 scene_name=bridge_table_1_v1 robot=widowx overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png" robot_x=0.147 robot_y=0.028 ;; carrot) env_name=PutCarrotOnPlateInScene-v0 scene_name=bridge_table_1_v1 robot=widowx overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png" robot_x=0.147 robot_y=0.028 ;; stack) env_name=StackGreenCubeOnYellowCubeBakedTexInScene-v0 scene_name=bridge_table_1_v1 robot=widowx overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png" robot_x=0.147 robot_y=0.028 ;; eggplant) env_name=PutEggplantInBasketScene-v0 scene_name=bridge_table_1_v2 robot=widowx_sink_camera_setup overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_sink.png" robot_x=0.127 robot_y=0.06 ;; esac mkdir -p "${panel_dir}" if [[ ! -f "${seed_dir}/result.json" ]]; then if [[ -e "${seed_dir}" ]]; then echo "refusing partial seed directory: ${seed_dir}" >&2 exit 2 fi mkdir -p "${seed_dir}" CUDA_VISIBLE_DEVICES="${gpu}" \ "${runtime}/.venv/bin/python3" deployment/model_server/server_policy.py \ --ckpt_path "${checkpoint}" \ --port "${port}" \ --idle_timeout -1 \ --use_bf16 \ --seed "${policy_seed}" \ >"${seed_dir}/policy_server.log" 2>&1 & server_pid=$! "${runtime}/.venv/bin/python" - "${port}" "${server_pid}" <<'PY' import socket import sys import time from pathlib import Path port = int(sys.argv[1]) server_pid = int(sys.argv[2]) deadline = time.monotonic() + 240 while time.monotonic() < deadline: if not Path(f"/proc/{server_pid}").exists(): raise SystemExit("policy server exited during startup") with socket.socket() as sock: sock.settimeout(1) if sock.connect_ex(("127.0.0.1", port)) == 0: raise SystemExit(0) time.sleep(2) raise SystemExit("policy server did not open its port") PY export CUDA_VISIBLE_DEVICES="${gpu}" export LD_LIBRARY_PATH="${runtime}/system-libs/libvulkan1/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" set +e "${runtime}/.venv-simpler/bin/python" -c ' import importlib import runpy evaluator = importlib.import_module("simpler_env.evaluation.maniskill2_evaluator") evaluator.write_video = lambda *args, **kwargs: None from examples.SimplerEnv.eval_files.model2simpler_interface import ModelClient ModelClient.visualize_epoch = lambda *args, **kwargs: None runpy.run_path("examples/SimplerEnv/eval_files/start_simpler_env.py", run_name="__main__") ' \ --ckpt-path "${checkpoint}" \ --port "${port}" \ --robot "${robot}" \ --policy-setup widowx_bridge \ --control-freq 5 \ --sim-freq 500 \ --max-episode-steps 120 \ --env-name "${env_name}" \ --scene-name "${scene_name}" \ --rgb-overlay-path "${overlay}" \ --robot-init-x-range "${robot_x}" "${robot_x}" 1 \ --robot-init-y-range "${robot_y}" "${robot_y}" 1 \ --obj-variation-mode episode \ --obj-episode-range 0 "${episodes_per_task}" \ --robot-init-rot-quat-center 0 0 0 1 \ --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \ --additional-env-save-tags "policy_seed_${policy_seed}_no_video" \ --logging-dir "${seed_dir}/discarded_artifact_paths" \ --result-json "${seed_dir}/result.json" \ >"${seed_dir}/client.log" 2>&1 client_exit=$? set -e kill "${server_pid}" 2>/dev/null || true wait "${server_pid}" 2>/dev/null || true server_pid="" printf '%s\n' "${client_exit}" >"${seed_dir}/exitcode" [[ "${client_exit}" == "0" ]] || exit "${client_exit}" fi "${runtime}/.venv/bin/python" - "${panel_dir}" "${step}" "${task}" "${policy_seed}" "${episodes_per_task}" <<'PY' import json import os import sys from pathlib import Path panel = Path(sys.argv[1]) step = int(sys.argv[2]) task = sys.argv[3] seed = int(sys.argv[4]) episodes = int(sys.argv[5]) result = json.loads((panel / f"seed_{seed}" / "result.json").read_text(encoding="utf-8")) successes = [bool(value) for value in result["successes"]] if len(successes) != episodes: raise RuntimeError(f"incomplete result for seed {seed}: {len(successes)} != {episodes}") summary = { "schema_version": 1, "status": "complete_no_video_strict_success_panel", "checkpoint_step": step, "task": task, "policy_seeds": [seed], "episodes_per_seed": episodes, "success_count": sum(successes), "rollout_count": episodes, "strict_success_rate": sum(successes) / episodes, "rows": [{"policy_seed": seed, "successes": successes, "success_count": sum(successes)}], } target = panel / "summary.json" temporary = target.with_suffix(f".json.tmp.{os.getpid()}") temporary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(temporary, target) print(json.dumps(summary, sort_keys=True)) PY } checkpoint="$(resolve_checkpoint | tail -n 1)" if [[ "${INFOFLOW_EVAL_VALIDATE_ONLY:-0}" == "1" ]]; then printf 'arm=%s\nstep=%s\ngpu=%s\nport=%s\ncheckpoint=%s\noutput=%s\n' \ "${arm}" "${step}" "${gpu}" "${port}" "${checkpoint}" "${step_dir}" exit 0 fi cd "${project}" for task in stack carrot spoon eggplant; do run_panel "${checkpoint}" "${task}" done "${runtime}/.venv/bin/python" - "${step_dir}" "${arm}" "${step}" "${policy_seed}" "${episodes_per_task}" <<'PY' import json import os import sys from pathlib import Path step_dir = Path(sys.argv[1]) arm = sys.argv[2] step = int(sys.argv[3]) seed = int(sys.argv[4]) episodes = int(sys.argv[5]) tasks = ("stack", "carrot", "spoon", "eggplant") rows = {} for task in tasks: summary = json.loads((step_dir / task / "summary.json").read_text(encoding="utf-8")) rows[task] = { "success_count": summary["success_count"], "rollout_count": summary["rollout_count"], "strict_success_rate": summary["strict_success_rate"], } total_success = sum(row["success_count"] for row in rows.values()) total_rollouts = sum(row["rollout_count"] for row in rows.values()) result = { "schema_version": 1, "status": "complete_no_video_four_task96", "arm": arm, "checkpoint_step": step, "policy_seed": seed, "episodes_per_task": episodes, "tasks": rows, "total_success": total_success, "total_rollouts": total_rollouts, "micro_success_rate": total_success / total_rollouts, "macro_success_rate": sum(row["strict_success_rate"] for row in rows.values()) / len(rows), } target = step_dir / "four_task_summary.json" temporary = target.with_suffix(f".json.tmp.{os.getpid()}") temporary.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.replace(temporary, target) print(json.dumps(result, sort_keys=True)) PY