File size: 2,181 Bytes
28ed187 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | #!/usr/bin/env bash
# Evaluate checkpoint-1500 as soon as it is complete, same settings as the other runs:
# policy_seed 12345, seed_base 42, 4 workers (42..45), the 160-episode exact-replay set.
# GPUs 4-7 only -- 2,3 are training and 0,1 belong to someone else.
set -u
R=/data/minha/pose6daug/train_robocasa/runs/pnpcountertocab_actaug_256_newemb_from60k_gbs64_30000steps_20260922_140904
CK=$R/checkpoint-1500
E=/lp-dev/jonghoon/___neurips_pose6daug___/migrator/code/06_eval/eval_groot15_exact_replay.sh
M=/data/minha/pose6daug/train_robocasa/myGR00T
L=/tmp/claude-1001/-home-nvidia/68c27ffb-706b-4cb3-8bf2-4618a200d378/scratchpad/eval_actaug_1500.log
echo "[wait] checkpoint-1500 대기 $(date -u +%H:%M:%S)" > "$L"
until [ -f "$CK/model.safetensors.index.json" ]; do sleep 60; done
sleep 60 # 쓰기 완료 여유
echo "[wait] 감지 $(date -u +%H:%M:%S)" >> "$L"
# 4 servers land 2-per-GPU on the cards the training is using; an OOM here would take
# the training down too, so wait until each card really has room.
NEED=42000
while true; do
f2=$(nvidia-smi -i 2 --query-gpu=memory.free --format=csv,noheader,nounits)
f3=$(nvidia-smi -i 3 --query-gpu=memory.free --format=csv,noheader,nounits)
[ "$f2" -ge "$NEED" ] && [ "$f3" -ge "$NEED" ] && break
echo "$(date -u +%H:%M:%S) 대기: GPU2 ${f2}MiB GPU3 ${f3}MiB (필요 ${NEED})" >> "$L"
sleep 300
done
echo "[wait] 메모리 확보, eval 시작 $(date -u +%H:%M:%S)" >> "$L"
MODEL_PATH="$CK" MYGROOT_ROOT="$M" EVAL_CLIENT="$M/scripts/eval_robocasa_replay_state_grasp.py" DATASETNAME=actaug_256 EVAL_OUTPUT_BASE=/home/nvidia/minha/pose6daug/evals/results EVAL_RUN_NAME=seed12345/actaug256_ckpt1500_exact160 REPLAY_STATE_ROOT=/lp-dev/jonghoon/isaac-gr00t/eval_results/pnpcountertocab_mimicgen8_exact160_replay_policyseed12345_20260506 N_EPISODES=160 EXPECTED_REPLAY_STATES=160 GPUS=2,3,2,3 POLICY_SEED=12345 SEED_BASE=42 bash "$E" >> "$L" 2>&1
echo "[wait] eval 종료 $(date -u +%H:%M:%S)" >> "$L"
python3 /tmp/claude-1001/-home-nvidia/68c27ffb-706b-4cb3-8bf2-4618a200d378/scratchpad/organize_videos.py --root /home/nvidia/minha/pose6daug/evals/results/seed12345/actaug256_ckpt1500_exact160 >> "$L" 2>&1
|