#!/usr/bin/env bash # A100 (Autoresearch3): run the whole mid-train pipeline unattended, in order: # 1. wait for the shard download (tmux dex_fetch_midtrain) to finish 2. norm stats of the 10-batch mixture # 3. prompt-len + one smoke batch through the loader (abort on failure) 4. wait until GPUs 1-6 are free # 5. launch training: pi05_base, 54-D interleaved, --freeze-vlm, --balance-tasks, batch 256 on 6 GPUs (FSDP 6), # 30K steps, cosine warmup 1K peak 2.5e-5 -> 2.5e-6 (openpi defaults), checkpoints every 5K. # Log: $SYT/logs/midtrain_launch.log; training log $SYT/logs/train_.log. (2026-10-24, decided with Kevin) set -uo pipefail source "${SYT:-/share/project/zhiwei/syt}/env.sh" EXP=${EXP:-dexhand_midtrain_v1} BATCHES=${BATCHES:-"cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3 pack_v2 dolls_v2"} STATS=${STATS:-dexhand_midtrain} TGPUS=${TGPUS:-2,3,4,5}; TFSDP=${TFSDP:-4}; TACCUM=${TACCUM:-2} # v1 actually ran on 2-5, FSDP 4, accum 2 (256 / 6 GPUs does not divide) REPO=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); REPO=${REPO%+} # cover_m2 is *_7seg_* OPENPI="$SYT/code/XPolicyLab/policy/Pi_05/openpi"; RUN="$SYT/code/dexhand/train/run.py" log() { echo "$(date -u +%m-%d\ %H:%MZ) $*"; } while tmux has-session -t dex_fetch_midtrain 2>/dev/null; do sleep 120; done log "download finished: $(ls -d $SYT/lerobot/dexhand/*seg_* | wc -l) shard dirs" cd "$OPENPI" if [ ! -f "$SYT/assets/$STATS/dual_marvin_wuji/norm_stats.json" ]; then log "norm stats ..."; JAX_PLATFORMS=cpu .venv/bin/python "$RUN" norm-stats-parquet --repo-id "$REPO" --stats "$STATS" \ > "$SYT/logs/midtrain_normstats.log" 2>&1 || { log "norm stats FAILED, see logs/midtrain_normstats.log"; exit 1; } fi log "norm stats ok" JAX_PLATFORMS=cpu .venv/bin/python "$RUN" prompt-len --repo-id "$REPO" --samples 400 > "$SYT/logs/midtrain_promptlen.log" 2>&1 \ || log "prompt-len failed (non-fatal), see logs/midtrain_promptlen.log" tail -3 "$SYT/logs/midtrain_promptlen.log" JAX_PLATFORMS=cpu .venv/bin/python "$RUN" smoke --repo-id "$REPO" --stats "$STATS" --balance-tasks --batch-size 8 \ > "$SYT/logs/midtrain_smoke.log" 2>&1 || { log "smoke FAILED, see logs/midtrain_smoke.log"; exit 1; } log "smoke ok" while true; do busy=$(nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits -i $TGPUS | awk -F", " '$2 > 1000 {print $1}' | tr "\n" " ") [ -z "$busy" ] && break log "waiting for GPUs $TGPUS (busy: $busy)"; sleep 300 done log "GPUs $TGPUS free -> launching $EXP" cd "$SYT/code/dexhand/train" GPUS=$TGPUS FSDP=$TFSDP BATCH=256 ACCUM=$TACCUM REPO="$REPO" bash train.sh "$EXP" 30000 5000 --stats "$STATS" --balance-tasks --freeze-vlm log "launched; log $SYT/logs/train_$EXP.log"