ReTime / scripts /evaluate_bridge_seed96_no_video.sh
mmmmmcyaaa's picture
Add Florence-Bridge code and handoff documentation
977b06e verified
Raw History Blame Contribute Delete
10.4 kB
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 7 ]]; then
echo "usage: $0 <run-dir> <step> <gpu> <port> <policy-seed> <arm> <output-root>" >&2
exit 64
fi
run_dir=$1
step=$2
gpu=$3
port=$4
policy_seed=$5
arm=$6
eval_root=$7
project="${OPTR_PROJECT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
runtime="${OPTR_RUNTIME_ROOT:?set OPTR_RUNTIME_ROOT to the runtime environment root}"
simpler_root="${SIMPLER_ROOT:-${runtime}/SimplerEnv}"
episodes_per_task=24
output_root="${eval_root}/policy_seed_${policy_seed}"
step_dir="${output_root}/step_${step}"
server_pid=""
mkdir -p "${step_dir}"
trap 'code=$?; trap - EXIT; if [[ -n "${server_pid}" ]]; then kill "${server_pid}" 2>/dev/null || true; wait "${server_pid}" 2>/dev/null || true; fi; printf "%s\n" "${code}" >"${step_dir}/exitcode.tmp.$$"; mv -f "${step_dir}/exitcode.tmp.$$" "${step_dir}/exitcode"; exit "${code}"' EXIT
resolve_checkpoint() {
"${runtime}/.venv/bin/python" - "${run_dir}" "${step}" <<'PY'
import json
import os
import sys
from pathlib import Path
run_dir = Path(sys.argv[1])
step = int(sys.argv[2])
checkpoint_dir = run_dir / "checkpoints" / f"steps_{step}_accelerate"
commit_path = checkpoint_dir / "checkpoint.json"
commit = json.loads(commit_path.read_text(encoding="utf-8"))
if commit["completed_steps"] != step or commit["world_size"] != 2:
raise RuntimeError(f"checkpoint identity mismatch: {checkpoint_dir}")
for entry in commit["files"]:
path = checkpoint_dir / entry["name"]
if not path.is_file() or path.stat().st_size != entry["bytes"]:
raise RuntimeError(f"incomplete checkpoint file: {path}")
source_record = next(entry for entry in commit["files"] if entry["name"] == "pytorch_model.pt")
source = checkpoint_dir / source_record["name"]
destination = run_dir / "checkpoints" / f"steps_{step}_pytorch_model.pt"
try:
os.link(source, destination)
except FileExistsError:
# Parallel policy seeds share this verified compatibility link.
pass
metadata = {
"schema_version": 1,
"completed_steps": step,
"world_size": 2,
"bytes": destination.stat().st_size,
}
sidecar = Path(f"{destination}.json")
temporary = Path(f"{sidecar}.tmp.{os.getpid()}")
temporary.write_text(json.dumps(metadata, indent=2, sort_keys=True) + "\n", encoding="utf-8")
os.replace(temporary, sidecar)
print(destination)
PY
}
validate_task_summary() {
local summary=$1
local task=$2
"${runtime}/.venv/bin/python" - "${summary}" "${step}" "${task}" "${policy_seed}" "${episodes_per_task}" <<'PY'
import json
import sys
from pathlib import Path
summary = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
step = int(sys.argv[2])
task = sys.argv[3]
seed = int(sys.argv[4])
episodes = int(sys.argv[5])
if (
summary.get("status") != "complete_no_video_strict_success_panel"
or summary.get("checkpoint_step") != step
or summary.get("task") != task
or summary.get("policy_seeds") != [seed]
or summary.get("rollout_count") != episodes
):
raise RuntimeError(f"invalid task summary: {sys.argv[1]}")
PY
}
run_panel() {
local checkpoint=$1
local task=$2
local panel_dir="${step_dir}/${task}"
local seed_dir="${panel_dir}/seed_${policy_seed}"
if [[ -f "${panel_dir}/summary.json" ]]; then
validate_task_summary "${panel_dir}/summary.json" "${task}"
return
fi
case "${task}" in
spoon)
env_name=PutSpoonOnTableClothInScene-v0
scene_name=bridge_table_1_v1
robot=widowx
overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png"
robot_x=0.147
robot_y=0.028
;;
carrot)
env_name=PutCarrotOnPlateInScene-v0
scene_name=bridge_table_1_v1
robot=widowx
overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png"
robot_x=0.147
robot_y=0.028
;;
stack)
env_name=StackGreenCubeOnYellowCubeBakedTexInScene-v0
scene_name=bridge_table_1_v1
robot=widowx
overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png"
robot_x=0.147
robot_y=0.028
;;
eggplant)
env_name=PutEggplantInBasketScene-v0
scene_name=bridge_table_1_v2
robot=widowx_sink_camera_setup
overlay="${simpler_root}/ManiSkill2_real2sim/data/real_inpainting/bridge_sink.png"
robot_x=0.127
robot_y=0.06
;;
esac
mkdir -p "${panel_dir}"
if [[ ! -f "${seed_dir}/result.json" ]]; then
if [[ -e "${seed_dir}" ]]; then
echo "refusing partial seed directory: ${seed_dir}" >&2
exit 2
fi
mkdir -p "${seed_dir}"
CUDA_VISIBLE_DEVICES="${gpu}" \
"${runtime}/.venv/bin/python3" deployment/model_server/server_policy.py \
--ckpt_path "${checkpoint}" \
--port "${port}" \
--idle_timeout -1 \
--use_bf16 \
--seed "${policy_seed}" \
>"${seed_dir}/policy_server.log" 2>&1 &
server_pid=$!
"${runtime}/.venv/bin/python" - "${port}" "${server_pid}" <<'PY'
import socket
import sys
import time
from pathlib import Path
port = int(sys.argv[1])
server_pid = int(sys.argv[2])
deadline = time.monotonic() + 240
while time.monotonic() < deadline:
if not Path(f"/proc/{server_pid}").exists():
raise SystemExit("policy server exited during startup")
with socket.socket() as sock:
sock.settimeout(1)
if sock.connect_ex(("127.0.0.1", port)) == 0:
raise SystemExit(0)
time.sleep(2)
raise SystemExit("policy server did not open its port")
PY
export CUDA_VISIBLE_DEVICES="${gpu}"
export LD_LIBRARY_PATH="${runtime}/system-libs/libvulkan1/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}"
set +e
"${runtime}/.venv-simpler/bin/python" -c '
import importlib
import runpy
evaluator = importlib.import_module("simpler_env.evaluation.maniskill2_evaluator")
evaluator.write_video = lambda *args, **kwargs: None
from examples.SimplerEnv.eval_files.model2simpler_interface import ModelClient
ModelClient.visualize_epoch = lambda *args, **kwargs: None
runpy.run_path("examples/SimplerEnv/eval_files/start_simpler_env.py", run_name="__main__")
' \
--ckpt-path "${checkpoint}" \
--port "${port}" \
--robot "${robot}" \
--policy-setup widowx_bridge \
--control-freq 5 \
--sim-freq 500 \
--max-episode-steps 120 \
--env-name "${env_name}" \
--scene-name "${scene_name}" \
--rgb-overlay-path "${overlay}" \
--robot-init-x-range "${robot_x}" "${robot_x}" 1 \
--robot-init-y-range "${robot_y}" "${robot_y}" 1 \
--obj-variation-mode episode \
--obj-episode-range 0 "${episodes_per_task}" \
--robot-init-rot-quat-center 0 0 0 1 \
--robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
--additional-env-save-tags "policy_seed_${policy_seed}_no_video" \
--logging-dir "${seed_dir}/discarded_artifact_paths" \
--result-json "${seed_dir}/result.json" \
>"${seed_dir}/client.log" 2>&1
client_exit=$?
set -e
kill "${server_pid}" 2>/dev/null || true
wait "${server_pid}" 2>/dev/null || true
server_pid=""
printf '%s\n' "${client_exit}" >"${seed_dir}/exitcode"
[[ "${client_exit}" == "0" ]] || exit "${client_exit}"
fi
"${runtime}/.venv/bin/python" - "${panel_dir}" "${step}" "${task}" "${policy_seed}" "${episodes_per_task}" <<'PY'
import json
import os
import sys
from pathlib import Path
panel = Path(sys.argv[1])
step = int(sys.argv[2])
task = sys.argv[3]
seed = int(sys.argv[4])
episodes = int(sys.argv[5])
result = json.loads((panel / f"seed_{seed}" / "result.json").read_text(encoding="utf-8"))
successes = [bool(value) for value in result["successes"]]
if len(successes) != episodes:
raise RuntimeError(f"incomplete result for seed {seed}: {len(successes)} != {episodes}")
summary = {
"schema_version": 1,
"status": "complete_no_video_strict_success_panel",
"checkpoint_step": step,
"task": task,
"policy_seeds": [seed],
"episodes_per_seed": episodes,
"success_count": sum(successes),
"rollout_count": episodes,
"strict_success_rate": sum(successes) / episodes,
"rows": [{"policy_seed": seed, "successes": successes, "success_count": sum(successes)}],
}
target = panel / "summary.json"
temporary = target.with_suffix(f".json.tmp.{os.getpid()}")
temporary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
os.replace(temporary, target)
print(json.dumps(summary, sort_keys=True))
PY
}
checkpoint="$(resolve_checkpoint | tail -n 1)"
if [[ "${INFOFLOW_EVAL_VALIDATE_ONLY:-0}" == "1" ]]; then
printf 'arm=%s\nstep=%s\ngpu=%s\nport=%s\ncheckpoint=%s\noutput=%s\n' \
"${arm}" "${step}" "${gpu}" "${port}" "${checkpoint}" "${step_dir}"
exit 0
fi
cd "${project}"
for task in stack carrot spoon eggplant; do
run_panel "${checkpoint}" "${task}"
done
"${runtime}/.venv/bin/python" - "${step_dir}" "${arm}" "${step}" "${policy_seed}" "${episodes_per_task}" <<'PY'
import json
import os
import sys
from pathlib import Path
step_dir = Path(sys.argv[1])
arm = sys.argv[2]
step = int(sys.argv[3])
seed = int(sys.argv[4])
episodes = int(sys.argv[5])
tasks = ("stack", "carrot", "spoon", "eggplant")
rows = {}
for task in tasks:
summary = json.loads((step_dir / task / "summary.json").read_text(encoding="utf-8"))
rows[task] = {
"success_count": summary["success_count"],
"rollout_count": summary["rollout_count"],
"strict_success_rate": summary["strict_success_rate"],
}
total_success = sum(row["success_count"] for row in rows.values())
total_rollouts = sum(row["rollout_count"] for row in rows.values())
result = {
"schema_version": 1,
"status": "complete_no_video_four_task96",
"arm": arm,
"checkpoint_step": step,
"policy_seed": seed,
"episodes_per_task": episodes,
"tasks": rows,
"total_success": total_success,
"total_rollouts": total_rollouts,
"micro_success_rate": total_success / total_rollouts,
"macro_success_rate": sum(row["strict_success_rate"] for row in rows.values()) / len(rows),
}
target = step_dir / "four_task_summary.json"
temporary = target.with_suffix(f".json.tmp.{os.getpid()}")
temporary.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
os.replace(temporary, target)
print(json.dumps(result, sort_keys=True))
PY