File size: 2,786 Bytes
6a08a28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#!/usr/bin/env bash
# A100 (Autoresearch3): run the whole mid-train pipeline unattended, in order:
#   1. wait for the shard download (tmux dex_fetch_midtrain) to finish      2. norm stats of the 10-batch mixture
#   3. prompt-len + one smoke batch through the loader (abort on failure)      4. wait until GPUs 1-6 are free
#   5. launch training: pi05_base, 54-D interleaved, --freeze-vlm, --balance-tasks, batch 256 on 6 GPUs (FSDP 6),
#      30K steps, cosine warmup 1K peak 2.5e-5 -> 2.5e-6 (openpi defaults), checkpoints every 5K.
# Log: $SYT/logs/midtrain_launch.log; training log $SYT/logs/train_<exp>.log.   (2026-10-24, decided with Kevin)
set -uo pipefail
source "${SYT:-/share/project/zhiwei/syt}/env.sh"
EXP=${EXP:-dexhand_midtrain_v1}
BATCHES=${BATCHES:-"cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3 pack_v2 dolls_v2"}
STATS=${STATS:-dexhand_midtrain}
TGPUS=${TGPUS:-2,3,4,5}; TFSDP=${TFSDP:-4}; TACCUM=${TACCUM:-2}   # v1 actually ran on 2-5, FSDP 4, accum 2 (256 / 6 GPUs does not divide)
REPO=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); REPO=${REPO%+}    # cover_m2 is *_7seg_*
OPENPI="$SYT/code/XPolicyLab/policy/Pi_05/openpi"; RUN="$SYT/code/dexhand/train/run.py"
log() { echo "$(date -u +%m-%d\ %H:%MZ) $*"; }
while tmux has-session -t dex_fetch_midtrain 2>/dev/null; do sleep 120; done
log "download finished: $(ls -d $SYT/lerobot/dexhand/*seg_* | wc -l) shard dirs"
cd "$OPENPI"
if [ ! -f "$SYT/assets/$STATS/dual_marvin_wuji/norm_stats.json" ]; then
    log "norm stats ..."; JAX_PLATFORMS=cpu .venv/bin/python "$RUN" norm-stats-parquet --repo-id "$REPO" --stats "$STATS" \
        > "$SYT/logs/midtrain_normstats.log" 2>&1 || { log "norm stats FAILED, see logs/midtrain_normstats.log"; exit 1; }
fi
log "norm stats ok"
JAX_PLATFORMS=cpu .venv/bin/python "$RUN" prompt-len --repo-id "$REPO" --samples 400 > "$SYT/logs/midtrain_promptlen.log" 2>&1 \
    || log "prompt-len failed (non-fatal), see logs/midtrain_promptlen.log"
tail -3 "$SYT/logs/midtrain_promptlen.log"
JAX_PLATFORMS=cpu .venv/bin/python "$RUN" smoke --repo-id "$REPO" --stats "$STATS" --balance-tasks --batch-size 8 \
    > "$SYT/logs/midtrain_smoke.log" 2>&1 || { log "smoke FAILED, see logs/midtrain_smoke.log"; exit 1; }
log "smoke ok"
while true; do
    busy=$(nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits -i $TGPUS | awk -F", " '$2 > 1000 {print $1}' | tr "\n" " ")
    [ -z "$busy" ] && break
    log "waiting for GPUs $TGPUS (busy: $busy)"; sleep 300
done
log "GPUs $TGPUS free -> launching $EXP"
cd "$SYT/code/dexhand/train"
GPUS=$TGPUS FSDP=$TFSDP BATCH=256 ACCUM=$TACCUM REPO="$REPO" bash train.sh "$EXP" 30000 5000 --stats "$STATS" --balance-tasks --freeze-vlm
log "launched; log $SYT/logs/train_$EXP.log"