#!/usr/bin/env bash # Launch dexhand pi0.5 training (train/run.py train) in the background on the A100 box; adapted from the gripper # project's $SYT/code/runner/train.sh. # Usage: [GPUS=.. FSDP=.. BATCH=.. ACCUM=.. REPO=..] bash train.sh [extra run.py args] # probe: bash train.sh probe 120 100000 --decay-steps 2000 --overwrite --fake-data # real: bash train.sh dexhand_cover_v2_ilv 20000 5000 # resume: same command + --resume # Log: $SYT/logs/train_.log Checkpoints: $SYT/checkpoints/dexhand_ilv// set -euo pipefail source "${SYT:-/share/project/zhiwei/syt}/env.sh" exp=$1; steps=$2; save=$3; shift 3 gpus=${GPUS:-0,1,2,3,4,5,6,7}; fsdp=${FSDP:-8}; batch=${BATCH:-256}; accum=${ACCUM:-1} repo=${REPO:-glob:dexhand/cover_v2_7seg_*} cd "$SYT/code/XPolicyLab/policy/Pi_05/openpi" export OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 NUMEXPR_NUM_THREADS=1 # one thread per data worker export PYTHONUNBUFFERED=1 CUDA_VISIBLE_DEVICES=$gpus log="$SYT/logs/train_${exp}.log" nohup .venv/bin/python "$SYT/code/dexhand/train/run.py" train --repo-id "$repo" --exp-name "$exp" \ --steps "$steps" --save-interval "$save" --fsdp "$fsdp" --batch-size "$batch" --num-workers 64 --log-interval 10 \ --grad-accum "$accum" "$@" > "$log" 2>&1 & echo "started pid $! on GPUs $gpus (fsdp $fsdp, batch $batch, grad-accum $accum) -> $log"