HI
actaug GR00T train+eval bundle: code (save-at/stop-at), eval stack (porting fixes), subset builder, protocol docs
38dc077 verified | # GR00T actaug training: 2 GPUs, 30k-horizon schedule, stop at STOP_AT, save at SAVE_AT. | |
| # Driven by env vars (--export=ALL,...): DATASET, OUT_DIR, SAVE_AT, STOP_AT, MAX_STEPS, BATCH, MODEL_OUTPUT_DIR | |
| #SBATCH -p sjw_alinlab_premium,background | |
| #SBATCH --gres=gpu:2 | |
| #SBATCH --time=12:00:00 | |
| #SBATCH --requeue | |
| #SBATCH -o /sjw_alinlab/home/jonghoon/pose6daug/hf_model/myGR00T/logs/actaug_%j.out | |
| #SBATCH -e /sjw_alinlab/home/jonghoon/pose6daug/hf_model/myGR00T/logs/actaug_%j.err | |
| set -uo pipefail | |
| : "${DATASET:?}"; : "${OUT_DIR:?}"; : "${SAVE_AT:?}" | |
| STOP_AT="${STOP_AT:-10000}"; MAX_STEPS="${MAX_STEPS:-30000}"; BATCH="${BATCH:-32}" | |
| MYGROOT=/sjw_alinlab/home/jonghoon/pose6daug/hf_model/myGR00T | |
| BASE=/sjw_alinlab/home/jonghoon/pose6daug/hf_model/base | |
| mkdir -p "$OUT_DIR" "$MYGROOT/logs" | |
| source ~/miniconda3/etc/profile.d/conda.sh; conda activate mygr00t | |
| export PYTHONPATH="$MYGROOT${PYTHONPATH:+:$PYTHONPATH}" | |
| export NO_ALBUMENTATIONS_UPDATE=1 TOKENIZERS_PARALLELISM=false HF_HUB_OFFLINE=1 | |
| cd "$MYGROOT" | |
| echo "### actaug train ###"; hostname; date; nvidia-smi -L | |
| # dataset-ready guard: wait until all videos present (download may still be finishing) | |
| EXPECTED_VIDS="${EXPECTED_VIDS:-3072}" | |
| for i in $(seq 1 180); do | |
| n=$(find -L "$DATASET/videos" -name '*.mp4' 2>/dev/null | wc -l) | |
| [ "$n" -ge "$EXPECTED_VIDS" ] && { echo "[dataset-ready] videos $n/$EXPECTED_VIDS"; break; } | |
| echo "[wait-dataset] videos $n/$EXPECTED_VIDS ..."; sleep 30 | |
| done | |
| echo "DATASET=$DATASET OUT=$OUT_DIR BATCH=$BATCH x 2gpu (global $((BATCH*2))) MAX_STEPS=$MAX_STEPS STOP_AT=$STOP_AT" | |
| echo "SAVE_AT=$SAVE_AT" | |
| # resume-on-preemption: if a checkpoint already exists (from a preempted+requeued run), resume from newest | |
| RESUME_ARG="" | |
| if compgen -G "$OUT_DIR/checkpoint-*" >/dev/null 2>&1; then RESUME_ARG="--resume"; echo "[resume] existing checkpoint found -> resuming"; else echo "[resume] no checkpoint yet -> fresh start from base"; fi | |
| python scripts/gr00t_finetune.py \ | |
| --dataset-path "$DATASET" --output-dir "$OUT_DIR" \ | |
| --data-config single_panda_gripper --embodiment-tag new_embodiment \ | |
| --backbone-model-type eagle --backbone-select-layer 12 --base-model-path "$BASE" \ | |
| --num-gpus 2 --batch-size "$BATCH" --gradient-accumulation-steps 1 \ | |
| --max-steps "$MAX_STEPS" --stop-at "$STOP_AT" --save-at "$SAVE_AT" \ | |
| --dataloader-num-workers 8 --report-to tensorboard $RESUME_ARG | |
| echo "### done rc=$? ###"; date | |