Download code/dexhand/train/midtrain_launch.sh from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 2.79 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/dexhand/train/midtrain_launch.sh
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/dexhand/train/midtrain_launch.sh
-
curl -L -o midtrain_launch.sh https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/dexhand/train/midtrain_launch.sh
2.79 kB
| # A100 (Autoresearch3): run the whole mid-train pipeline unattended, in order: | |
| # 1. wait for the shard download (tmux dex_fetch_midtrain) to finish 2. norm stats of the 10-batch mixture | |
| # 3. prompt-len + one smoke batch through the loader (abort on failure) 4. wait until GPUs 1-6 are free | |
| # 5. launch training: pi05_base, 54-D interleaved, --freeze-vlm, --balance-tasks, batch 256 on 6 GPUs (FSDP 6), | |
| # 30K steps, cosine warmup 1K peak 2.5e-5 -> 2.5e-6 (openpi defaults), checkpoints every 5K. | |
| # Log: $SYT/logs/midtrain_launch.log; training log $SYT/logs/train_<exp>.log. (2026-10-24, decided with Kevin) | |
| set -uo pipefail | |
| source "${SYT:-/share/project/zhiwei/syt}/env.sh" | |
| EXP=${EXP:-dexhand_midtrain_v1} | |
| BATCHES=${BATCHES:-"cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3 pack_v2 dolls_v2"} | |
| STATS=${STATS:-dexhand_midtrain} | |
| TGPUS=${TGPUS:-2,3,4,5}; TFSDP=${TFSDP:-4}; TACCUM=${TACCUM:-2} # v1 actually ran on 2-5, FSDP 4, accum 2 (256 / 6 GPUs does not divide) | |
| REPO=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); REPO=${REPO%+} # cover_m2 is *_7seg_* | |
| OPENPI="$SYT/code/XPolicyLab/policy/Pi_05/openpi"; RUN="$SYT/code/dexhand/train/run.py" | |
| log() { echo "$(date -u +%m-%d\ %H:%MZ) $*"; } | |
| while tmux has-session -t dex_fetch_midtrain 2>/dev/null; do sleep 120; done | |
| log "download finished: $(ls -d $SYT/lerobot/dexhand/*seg_* | wc -l) shard dirs" | |
| cd "$OPENPI" | |
| if [ ! -f "$SYT/assets/$STATS/dual_marvin_wuji/norm_stats.json" ]; then | |
| log "norm stats ..."; JAX_PLATFORMS=cpu .venv/bin/python "$RUN" norm-stats-parquet --repo-id "$REPO" --stats "$STATS" \ | |
| > "$SYT/logs/midtrain_normstats.log" 2>&1 || { log "norm stats FAILED, see logs/midtrain_normstats.log"; exit 1; } | |
| fi | |
| log "norm stats ok" | |
| JAX_PLATFORMS=cpu .venv/bin/python "$RUN" prompt-len --repo-id "$REPO" --samples 400 > "$SYT/logs/midtrain_promptlen.log" 2>&1 \ | |
| || log "prompt-len failed (non-fatal), see logs/midtrain_promptlen.log" | |
| tail -3 "$SYT/logs/midtrain_promptlen.log" | |
| JAX_PLATFORMS=cpu .venv/bin/python "$RUN" smoke --repo-id "$REPO" --stats "$STATS" --balance-tasks --batch-size 8 \ | |
| > "$SYT/logs/midtrain_smoke.log" 2>&1 || { log "smoke FAILED, see logs/midtrain_smoke.log"; exit 1; } | |
| log "smoke ok" | |
| while true; do | |
| busy=$(nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits -i $TGPUS | awk -F", " '$2 > 1000 {print $1}' | tr "\n" " ") | |
| [ -z "$busy" ] && break | |
| log "waiting for GPUs $TGPUS (busy: $busy)"; sleep 300 | |
| done | |
| log "GPUs $TGPUS free -> launching $EXP" | |
| cd "$SYT/code/dexhand/train" | |
| GPUS=$TGPUS FSDP=$TFSDP BATCH=256 ACCUM=$TACCUM REPO="$REPO" bash train.sh "$EXP" 30000 5000 --stats "$STATS" --balance-tasks --freeze-vlm | |
| log "launched; log $SYT/logs/train_$EXP.log" | |