dexhand-train-kit / code /dexhand /train /prepare_midtrain.sh
Noctis12138's picture
Add files using upload-large-folder tool
6a08a28 verified
Raw History Blame Contribute Delete
2.32 kB
#!/usr/bin/env bash
# A100 (Autoresearch3): fetch the model-v2 mid-training shards (public HF dataset Noctis12138/dexhand-midtrain via
# hf-mirror; re-run to pick up new ones) into $SYT/lerobot/dexhand/, then optionally compute the norm stats of the
# 10-batch mixture. Batches (2026-10-24): cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3
# pack_v2 dolls_v2 - 3,569 sub-task segments, 555K frames, 54-D joints, 25 fps, model v2.
# bash train/prepare_midtrain.sh fetch (resumable)
# bash train/prepare_midtrain.sh --stats also compute norm stats -> $SYT/assets/dexhand_midtrain
set -euo pipefail
source "${SYT:-/share/project/zhiwei/syt}/env.sh"
# mid-train v3 (2026-10-27); the v2 batches of mid-train v1 (cover_m2 align_v2 ... dolls_v2) were removed from HF that day
BATCHES=${BATCHES:-"cover_m3 align_m3 bottles_m3 blocks_m3 bowls_m3 pens_m3 pickup_m3a pickup_m3b pack_m3 dolls_m3"}
STATS=${STATS:-dexhand_midtrain_m3}
HF="$SYT/code/XPolicyLab/policy/Pi_05/openpi/.venv/bin/hf"
cd "$SYT/lerobot"
pats=(); for b in $BATCHES; do pats+=("lerobot/${b}_*seg_*/*"); done # cover_m2 shards are *_7seg_* (7-segment cut); ONE --include with all patterns: repeated
for try in 1 2 3 4 5 6; do # --include flags keep only the last one
if "$HF" download Noctis12138/dexhand-midtrain --repo-type dataset --include "${pats[@]}" --local-dir _hf_midtrain \
>> "$SYT/logs/fetch_midtrain.log" 2>&1; then
break
fi
echo "download attempt $try failed, retrying in 60 s"; sleep 60
done
mkdir -p dexhand
for d in _hf_midtrain/lerobot/*seg_*/; do n=$(basename "$d"); [ -e "dexhand/$n" ] || ln -s "$PWD/$d" "dexhand/$n"; done
echo "shards per batch:"; for b in $BATCHES; do echo " $b $(ls -d dexhand/${b}_*seg_* 2>/dev/null | wc -l)"; done
echo "segments: $(python3 -c "
import glob, json
bs='$BATCHES'.split()
print(sum(json.load(open(f))['total_episodes'] for b in bs for f in glob.glob(f'dexhand/{b}_*seg_*/meta/info.json')))")"
if [ "${1:-}" = "--stats" ]; then
cd "$SYT/code/XPolicyLab/policy/Pi_05/openpi"
pat=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); pat=${pat%+}
.venv/bin/python "$SYT/code/dexhand/train/run.py" norm-stats-parquet --repo-id "$pat" --stats "$STATS"
fi