File size: 3,079 Bytes
5d08972 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | #!/usr/bin/env bash
# Check everything run_aug32.sh needs before committing a multi-hour batch to a GPU.
# Exits non-zero if anything required is missing.
set -uo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VACE="$(cd "$HERE/.." && pwd)"
fail=0
ok() { printf ' \033[32mOK \033[0m %s\n' "$1"; }
bad() { printf ' \033[31mMISS\033[0m %s\n' "$1"; fail=1; }
have() { [ -e "$1" ] && ok "$2" || bad "$2 ($1)"; }
runs() { [ -x "$1" ] && ok "$2 ($("$1" -V 2>&1))" || bad "$2 ($1)"; }
echo "νκ²½ (κ°μνκ²½ β μλ‘ λ§λ€ νμ μμ)"
runs "${VACE_PY_ROBOCASA:-/lp-dev/jonghoon/robocasa_calib/envs/robocasa/bin/python}" "batch driver (robocasa, mujoco)"
runs "${VACE_PY_WAN:-/lp-dev/jonghoon/VACE/venv/bin/python}" "WAN inference (torch+flash_attn)"
have "${VACE_RENDER_SCRIPT_DIR:-$HERE/vendor}/render_mesh_views_side.py" "render_mesh_views_side.py (vendored)"
"${VACE_RENDER_PYTHON:-${VACE_PY_ROBOCASA:-/lp-dev/jonghoon/robocasa_calib/envs/robocasa/bin/python}}" \
-c "import trimesh,pyrender" 2>/dev/null && ok "trimesh + pyrender (mesh view rendering)" \
|| bad "trimesh + pyrender missing from the render python"
echo
echo "VACE μ½λμ κ°μ€μΉ"
have "$VACE/VACE/vace/vace_wan_inference.py" "VACE fork (ali-vilab/VACE + batch flags)"
have "$VACE/VACE/models/Wan2.1-VACE-1.3B" "Wan2.1-VACE-1.3B weights (via models symlink)"
have "$VACE/VACE/models/VACE-Annotators" "VACE-Annotators"
have "$VACE/vace_object_insert_batch.py" "batch driver"
echo
echo "λ°μ΄ν°μ
"
have "${VACE_ROBOCASA_ROOT:-/home/nvidia/jonghoon/robocasa_full/pickplace_target_human}" "μμ€ μνΌμλ (μμ + extras)"
have "${VACE_AUG_MATERIALS:-/home/nvidia/jonghoon/robocasa_full/aug_materials/robocasa}" "GT target-object λ§μ€ν¬"
for r in $(echo "${VACE_ASSET_ROOTS:-/lp-dev/jonghoon/robocasa_calib/repos/robocasa/robocasa/models/assets/objects/lightwheel:/lp-dev/jonghoon/robocasa_calib/repos/robocasa/robocasa/models/assets/objects/objaverse}" | tr ":" " "); do
have "$r" "λ©μ¬ $(basename "$r")"
done
echo
echo "GT λ§μ€ν¬ 컀λ²λ¦¬μ§ (prep-worker κ° μꡬ β μμΌλ©΄ κ·Έ μνΌμλλ μ€ν¨)"
A="${VACE_AUG_MATERIALS:-/home/nvidia/jonghoon/robocasa_full/aug_materials/robocasa}"
for t in PickPlaceCounterToCabinet PickPlaceCounterToStove PickPlaceDrawerToCounter \
PickPlaceSinkToCounter PickPlaceToasterToCounter; do
n=0
for d in "$A/$t"/ep*; do
[ -f "$d/mask_gt_target_object_robot0_agentview_left.mp4" ] &&
[ -f "$d/mask_gt_target_object_robot0_agentview_right.mp4" ] &&
[ -f "$d/mask_gt_target_object_robot0_eye_in_hand.mp4" ] && n=$((n+1))
done 2>/dev/null
printf ' %-30s %4d episodes\n' "$t" "$n"
done
echo
echo "GPU"
nvidia-smi --query-gpu=index,memory.used,memory.total,utilization.gpu --format=csv,noheader 2>/dev/null |
awk -F', ' '{printf " gpu %s %s / %s util %s\n",$1,$2,$3,$4}'
echo " νμ: μ½ 27 GB (Wan2.1-VACE-1.3B, --offload_model False κΈ°μ€)"
echo
[ $fail -eq 0 ] && echo "λͺ¨λ μ€λΉλ¨." || echo "λλ½ νλͺ© μμ β μ MISS νμΈ."
exit $fail
|