dexhand-train-kit / code /dexhand /train /midtrain_launch.sh
Noctis12138's picture
Add files using upload-large-folder tool
6a08a28 verified
Raw History Blame Contribute Delete
2.79 kB
#!/usr/bin/env bash
# A100 (Autoresearch3): run the whole mid-train pipeline unattended, in order:
# 1. wait for the shard download (tmux dex_fetch_midtrain) to finish 2. norm stats of the 10-batch mixture
# 3. prompt-len + one smoke batch through the loader (abort on failure) 4. wait until GPUs 1-6 are free
# 5. launch training: pi05_base, 54-D interleaved, --freeze-vlm, --balance-tasks, batch 256 on 6 GPUs (FSDP 6),
# 30K steps, cosine warmup 1K peak 2.5e-5 -> 2.5e-6 (openpi defaults), checkpoints every 5K.
# Log: $SYT/logs/midtrain_launch.log; training log $SYT/logs/train_<exp>.log. (2026-10-24, decided with Kevin)
set -uo pipefail
source "${SYT:-/share/project/zhiwei/syt}/env.sh"
EXP=${EXP:-dexhand_midtrain_v1}
BATCHES=${BATCHES:-"cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3 pack_v2 dolls_v2"}
STATS=${STATS:-dexhand_midtrain}
TGPUS=${TGPUS:-2,3,4,5}; TFSDP=${TFSDP:-4}; TACCUM=${TACCUM:-2} # v1 actually ran on 2-5, FSDP 4, accum 2 (256 / 6 GPUs does not divide)
REPO=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); REPO=${REPO%+} # cover_m2 is *_7seg_*
OPENPI="$SYT/code/XPolicyLab/policy/Pi_05/openpi"; RUN="$SYT/code/dexhand/train/run.py"
log() { echo "$(date -u +%m-%d\ %H:%MZ) $*"; }
while tmux has-session -t dex_fetch_midtrain 2>/dev/null; do sleep 120; done
log "download finished: $(ls -d $SYT/lerobot/dexhand/*seg_* | wc -l) shard dirs"
cd "$OPENPI"
if [ ! -f "$SYT/assets/$STATS/dual_marvin_wuji/norm_stats.json" ]; then
log "norm stats ..."; JAX_PLATFORMS=cpu .venv/bin/python "$RUN" norm-stats-parquet --repo-id "$REPO" --stats "$STATS" \
> "$SYT/logs/midtrain_normstats.log" 2>&1 || { log "norm stats FAILED, see logs/midtrain_normstats.log"; exit 1; }
fi
log "norm stats ok"
JAX_PLATFORMS=cpu .venv/bin/python "$RUN" prompt-len --repo-id "$REPO" --samples 400 > "$SYT/logs/midtrain_promptlen.log" 2>&1 \
|| log "prompt-len failed (non-fatal), see logs/midtrain_promptlen.log"
tail -3 "$SYT/logs/midtrain_promptlen.log"
JAX_PLATFORMS=cpu .venv/bin/python "$RUN" smoke --repo-id "$REPO" --stats "$STATS" --balance-tasks --batch-size 8 \
> "$SYT/logs/midtrain_smoke.log" 2>&1 || { log "smoke FAILED, see logs/midtrain_smoke.log"; exit 1; }
log "smoke ok"
while true; do
busy=$(nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits -i $TGPUS | awk -F", " '$2 > 1000 {print $1}' | tr "\n" " ")
[ -z "$busy" ] && break
log "waiting for GPUs $TGPUS (busy: $busy)"; sleep 300
done
log "GPUs $TGPUS free -> launching $EXP"
cd "$SYT/code/dexhand/train"
GPUS=$TGPUS FSDP=$TFSDP BATCH=256 ACCUM=$TACCUM REPO="$REPO" bash train.sh "$EXP" 30000 5000 --stats "$STATS" --balance-tasks --freeze-vlm
log "launched; log $SYT/logs/train_$EXP.log"