File size: 1,398 Bytes
6a08a28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | #!/usr/bin/env bash
# Launch dexhand pi0.5 training (train/run.py train) in the background on the A100 box; adapted from the gripper
# project's $SYT/code/runner/train.sh.
# Usage: [GPUS=.. FSDP=.. BATCH=.. ACCUM=.. REPO=..] bash train.sh <exp_name> <steps> <save_interval> [extra run.py args]
# probe: bash train.sh probe 120 100000 --decay-steps 2000 --overwrite --fake-data
# real: bash train.sh dexhand_cover_v2_ilv 20000 5000
# resume: same command + --resume
# Log: $SYT/logs/train_<exp>.log Checkpoints: $SYT/checkpoints/dexhand_ilv/<exp>/<step>
set -euo pipefail
source "${SYT:-/share/project/zhiwei/syt}/env.sh"
exp=$1; steps=$2; save=$3; shift 3
gpus=${GPUS:-0,1,2,3,4,5,6,7}; fsdp=${FSDP:-8}; batch=${BATCH:-256}; accum=${ACCUM:-1}
repo=${REPO:-glob:dexhand/cover_v2_7seg_*}
cd "$SYT/code/XPolicyLab/policy/Pi_05/openpi"
export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 # one thread per data worker
export PYTHONUNBUFFERED=1 CUDA_VISIBLE_DEVICES=$gpus
log="$SYT/logs/train_${exp}.log"
nohup .venv/bin/python "$SYT/code/dexhand/train/run.py" train --repo-id "$repo" --exp-name "$exp" \
--steps "$steps" --save-interval "$save" --fsdp "$fsdp" --batch-size "$batch" --num-workers 64 --log-interval 10 \
--grad-accum "$accum" "$@" > "$log" 2>&1 &
echo "started pid $! on GPUs $gpus (fsdp $fsdp, batch $batch, grad-accum $accum) -> $log"
|