Download scripts/evaluate_bridge_seed96_no_video.sh from mmmmmcyaaa/ReTime: direct link, hf CLI and curl.
- Browser
- Download file 10.4 kB
-
https://huggingface.co/mmmmmcyaaa/ReTime/resolve/main/scripts/evaluate_bridge_seed96_no_video.sh
- Command line
-
hf download hf://mmmmmcyaaa/ReTime/scripts/evaluate_bridge_seed96_no_video.sh
-
curl -L -o evaluate_bridge_seed96_no_video.sh https://huggingface.co/mmmmmcyaaa/ReTime/resolve/main/scripts/evaluate_bridge_seed96_no_video.sh
10.4 kB
| set -euo pipefail | |
| if [[ $# -ne 7 ]]; then | |
| echo "usage: $0 <run-dir> <step> <gpu> <port> <policy-seed> <arm> <output-root>" >&2 | |
| exit 64 | |
| fi | |
| run_dir=$1 | |
| step=$2 | |
| gpu=$3 | |
| port=$4 | |
| policy_seed=$5 | |
| arm=$6 | |
| eval_root=$7 | |
| project="${OPTR_PROJECT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}" | |
| runtime="${OPTR_RUNTIME_ROOT:?set OPTR_RUNTIME_ROOT to the runtime environment root}" | |
| simpler_root="${SIMPLER_ROOT:-${runtime}/SimplerEnv}" | |
| episodes_per_task=24 | |
| output_root="${eval_root}/policy_seed_${policy_seed}" | |
| step_dir="${output_root}/step_${step}" | |
| server_pid="" | |
| mkdir -p "${step_dir}" | |
| trap 'code=$?; trap - EXIT; if [[ -n "${server_pid}" ]]; then kill "${server_pid}" 2>/dev/null || true; wait "${server_pid}" 2>/dev/null || true; fi; printf "%s\n" "${code}" >"${step_dir}/exitcode.tmp.$$"; mv -f "${step_dir}/exitcode.tmp.$$" "${step_dir}/exitcode"; exit "${code}"' EXIT | |
| resolve_checkpoint() { | |
| "${runtime}/.venv/bin/python" - "${run_dir}" "${step}" <<'PY' | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| run_dir = Path(sys.argv[1]) | |
| step = int(sys.argv[2]) | |
| checkpoint_dir = run_dir / "checkpoints" / f"steps_{step}_accelerate" | |
| commit_path = checkpoint_dir / "checkpoint.json" | |
| commit = json.loads(commit_path.read_text(encoding="utf-8")) | |
| if commit["completed_steps"] != step or commit["world_size"] != 2: | |
| raise RuntimeError(f"checkpoint identity mismatch: {checkpoint_dir}") | |
| for entry in commit["files"]: | |
| path = checkpoint_dir / entry["name"] | |
| if not path.is_file() or path.stat().st_size != entry["bytes"]: | |
| raise RuntimeError(f"incomplete checkpoint file: {path}") | |
| source_record = next(entry for entry in commit["files"] if entry["name"] == "pytorch_model.pt") | |
| source = checkpoint_dir / source_record["name"] | |
| destination = run_dir / "checkpoints" / f"steps_{step}_pytorch_model.pt" | |
| try: | |
| os.link(source, destination) | |
| except FileExistsError: | |
| # Parallel policy seeds share this verified compatibility link. | |
| pass | |
| metadata = { | |
| "schema_version": 1, | |
| "completed_steps": step, | |
| "world_size": 2, | |
| "bytes": destination.stat().st_size, | |
| } | |
| sidecar = Path(f"{destination}.json") | |
| temporary = Path(f"{sidecar}.tmp.{os.getpid()}") | |
| temporary.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8") | |
| os.replace(temporary, sidecar) | |
| print(destination) | |
| PY | |
| } | |
| validate_task_summary() { | |
| local summary=$1 | |
| local task=$2 | |
| "${runtime}/.venv/bin/python" - "${summary}" "${step}" "${task}" "${policy_seed}" "${episodes_per_task}" <<'PY' | |
| import json | |
| import sys | |
| from pathlib import Path | |
| summary = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) | |
| step = int(sys.argv[2]) | |
| task = sys.argv[3] | |
| seed = int(sys.argv[4]) | |
| episodes = int(sys.argv[5]) | |
| if ( | |
| summary.get("status") != "complete_no_video_strict_success_panel" | |
| or summary.get("checkpoint_step") != step | |
| or summary.get("task") != task | |
| or summary.get("policy_seeds") != [seed] | |
| or summary.get("rollout_count") != episodes | |
| ): | |
| raise RuntimeError(f"invalid task summary: {sys.argv[1]}") | |
| PY | |
| } | |
| run_panel() { | |
| local checkpoint=$1 | |
| local task=$2 | |
| local panel_dir="${step_dir}/${task}" | |
| local seed_dir="${panel_dir}/seed_${policy_seed}" | |
| if [[ -f "${panel_dir}/summary.json" ]]; then | |
| validate_task_summary "${panel_dir}/summary.json" "${task}" | |
| return | |
| fi | |
| case "${task}" in | |
| spoon) | |
| env_name=PutSpoonOnTableClothInScene-v0 | |
| scene_name=bridge_table_1_v1 | |
| robot=widowx | |
| overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png" | |
| robot_x=0.147 | |
| robot_y=0.028 | |
| ;; | |
| carrot) | |
| env_name=PutCarrotOnPlateInScene-v0 | |
| scene_name=bridge_table_1_v1 | |
| robot=widowx | |
| overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png" | |
| robot_x=0.147 | |
| robot_y=0.028 | |
| ;; | |
| stack) | |
| env_name=StackGreenCubeOnYellowCubeBakedTexInScene-v0 | |
| scene_name=bridge_table_1_v1 | |
| robot=widowx | |
| overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png" | |
| robot_x=0.147 | |
| robot_y=0.028 | |
| ;; | |
| eggplant) | |
| env_name=PutEggplantInBasketScene-v0 | |
| scene_name=bridge_table_1_v2 | |
| robot=widowx_sink_camera_setup | |
| overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_sink.png" | |
| robot_x=0.127 | |
| robot_y=0.06 | |
| ;; | |
| esac | |
| mkdir -p "${panel_dir}" | |
| if [[ ! -f "${seed_dir}/result.json" ]]; then | |
| if [[ -e "${seed_dir}" ]]; then | |
| echo "refusing partial seed directory: ${seed_dir}" >&2 | |
| exit 2 | |
| fi | |
| mkdir -p "${seed_dir}" | |
| CUDA_VISIBLE_DEVICES="${gpu}" \ | |
| "${runtime}/.venv/bin/python3" deployment/model_server/server_policy.py \ | |
| --ckpt_path "${checkpoint}" \ | |
| --port "${port}" \ | |
| --idle_timeout -1 \ | |
| --use_bf16 \ | |
| --seed "${policy_seed}" \ | |
| >"${seed_dir}/policy_server.log" 2>&1 & | |
| server_pid=$! | |
| "${runtime}/.venv/bin/python" - "${port}" "${server_pid}" <<'PY' | |
| import socket | |
| import sys | |
| import time | |
| from pathlib import Path | |
| port = int(sys.argv[1]) | |
| server_pid = int(sys.argv[2]) | |
| deadline = time.monotonic() + 240 | |
| while time.monotonic() < deadline: | |
| if not Path(f"/proc/{server_pid}").exists(): | |
| raise SystemExit("policy server exited during startup") | |
| with socket.socket() as sock: | |
| sock.settimeout(1) | |
| if sock.connect_ex(("127.0.0.1", port)) == 0: | |
| raise SystemExit(0) | |
| time.sleep(2) | |
| raise SystemExit("policy server did not open its port") | |
| PY | |
| export CUDA_VISIBLE_DEVICES="${gpu}" | |
| export LD_LIBRARY_PATH="${runtime}/system-libs/libvulkan1/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" | |
| set +e | |
| "${runtime}/.venv-simpler/bin/python" -c ' | |
| import importlib | |
| import runpy | |
| evaluator = importlib.import_module("simpler_env.evaluation.maniskill2_evaluator") | |
| evaluator.write_video = lambda *args, **kwargs: None | |
| from examples.SimplerEnv.eval_files.model2simpler_interface import ModelClient | |
| ModelClient.visualize_epoch = lambda *args, **kwargs: None | |
| runpy.run_path("examples/SimplerEnv/eval_files/start_simpler_env.py", run_name="__main__") | |
| ' \ | |
| --ckpt-path "${checkpoint}" \ | |
| --port "${port}" \ | |
| --robot "${robot}" \ | |
| --policy-setup widowx_bridge \ | |
| --control-freq 5 \ | |
| --sim-freq 500 \ | |
| --max-episode-steps 120 \ | |
| --env-name "${env_name}" \ | |
| --scene-name "${scene_name}" \ | |
| --rgb-overlay-path "${overlay}" \ | |
| --robot-init-x-range "${robot_x}" "${robot_x}" 1 \ | |
| --robot-init-y-range "${robot_y}" "${robot_y}" 1 \ | |
| --obj-variation-mode episode \ | |
| --obj-episode-range 0 "${episodes_per_task}" \ | |
| --robot-init-rot-quat-center 0 0 0 1 \ | |
| --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \ | |
| --additional-env-save-tags "policy_seed_${policy_seed}_no_video" \ | |
| --logging-dir "${seed_dir}/discarded_artifact_paths" \ | |
| --result-json "${seed_dir}/result.json" \ | |
| >"${seed_dir}/client.log" 2>&1 | |
| client_exit=$? | |
| set -e | |
| kill "${server_pid}" 2>/dev/null || true | |
| wait "${server_pid}" 2>/dev/null || true | |
| server_pid="" | |
| printf '%s\n' "${client_exit}" >"${seed_dir}/exitcode" | |
| [[ "${client_exit}" == "0" ]] || exit "${client_exit}" | |
| fi | |
| "${runtime}/.venv/bin/python" - "${panel_dir}" "${step}" "${task}" "${policy_seed}" "${episodes_per_task}" <<'PY' | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| panel = Path(sys.argv[1]) | |
| step = int(sys.argv[2]) | |
| task = sys.argv[3] | |
| seed = int(sys.argv[4]) | |
| episodes = int(sys.argv[5]) | |
| result = json.loads((panel / f"seed_{seed}" / "result.json").read_text(encoding="utf-8")) | |
| successes = [bool(value) for value in result["successes"]] | |
| if len(successes) != episodes: | |
| raise RuntimeError(f"incomplete result for seed {seed}: {len(successes)} != {episodes}") | |
| summary = { | |
| "schema_version": 1, | |
| "status": "complete_no_video_strict_success_panel", | |
| "checkpoint_step": step, | |
| "task": task, | |
| "policy_seeds": [seed], | |
| "episodes_per_seed": episodes, | |
| "success_count": sum(successes), | |
| "rollout_count": episodes, | |
| "strict_success_rate": sum(successes) / episodes, | |
| "rows": [{"policy_seed": seed, "successes": successes, "success_count": sum(successes)}], | |
| } | |
| target = panel / "summary.json" | |
| temporary = target.with_suffix(f".json.tmp.{os.getpid()}") | |
| temporary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") | |
| os.replace(temporary, target) | |
| print(json.dumps(summary, sort_keys=True)) | |
| PY | |
| } | |
| checkpoint="$(resolve_checkpoint | tail -n 1)" | |
| if [[ "${INFOFLOW_EVAL_VALIDATE_ONLY:-0}" == "1" ]]; then | |
| printf 'arm=%s\nstep=%s\ngpu=%s\nport=%s\ncheckpoint=%s\noutput=%s\n' \ | |
| "${arm}" "${step}" "${gpu}" "${port}" "${checkpoint}" "${step_dir}" | |
| exit 0 | |
| fi | |
| cd "${project}" | |
| for task in stack carrot spoon eggplant; do | |
| run_panel "${checkpoint}" "${task}" | |
| done | |
| "${runtime}/.venv/bin/python" - "${step_dir}" "${arm}" "${step}" "${policy_seed}" "${episodes_per_task}" <<'PY' | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| step_dir = Path(sys.argv[1]) | |
| arm = sys.argv[2] | |
| step = int(sys.argv[3]) | |
| seed = int(sys.argv[4]) | |
| episodes = int(sys.argv[5]) | |
| tasks = ("stack", "carrot", "spoon", "eggplant") | |
| rows = {} | |
| for task in tasks: | |
| summary = json.loads((step_dir / task / "summary.json").read_text(encoding="utf-8")) | |
| rows[task] = { | |
| "success_count": summary["success_count"], | |
| "rollout_count": summary["rollout_count"], | |
| "strict_success_rate": summary["strict_success_rate"], | |
| } | |
| total_success = sum(row["success_count"] for row in rows.values()) | |
| total_rollouts = sum(row["rollout_count"] for row in rows.values()) | |
| result = { | |
| "schema_version": 1, | |
| "status": "complete_no_video_four_task96", | |
| "arm": arm, | |
| "checkpoint_step": step, | |
| "policy_seed": seed, | |
| "episodes_per_task": episodes, | |
| "tasks": rows, | |
| "total_success": total_success, | |
| "total_rollouts": total_rollouts, | |
| "micro_success_rate": total_success / total_rollouts, | |
| "macro_success_rate": sum(row["strict_success_rate"] for row in rows.values()) / len(rows), | |
| } | |
| target = step_dir / "four_task_summary.json" | |
| temporary = target.with_suffix(f".json.tmp.{os.getpid()}") | |
| temporary.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") | |
| os.replace(temporary, target) | |
| print(json.dumps(result, sort_keys=True)) | |
| PY | |