#!/usr/bin/env bash # A100 (Autoresearch3): fetch the model-v2 mid-training shards (public HF dataset Noctis12138/dexhand-midtrain via # hf-mirror; re-run to pick up new ones) into $SYT/lerobot/dexhand/, then optionally compute the norm stats of the # 10-batch mixture. Batches (2026-10-24): cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3 # pack_v2 dolls_v2 - 3,569 sub-task segments, 555K frames, 54-D joints, 25 fps, model v2. # bash train/prepare_midtrain.sh fetch (resumable) # bash train/prepare_midtrain.sh --stats also compute norm stats -> $SYT/assets/dexhand_midtrain set -euo pipefail source "${SYT:-/share/project/zhiwei/syt}/env.sh" # mid-train v3 (2026-10-27); the v2 batches of mid-train v1 (cover_m2 align_v2 ... dolls_v2) were removed from HF that day BATCHES=${BATCHES:-"cover_m3 align_m3 bottles_m3 blocks_m3 bowls_m3 pens_m3 pickup_m3a pickup_m3b pack_m3 dolls_m3"} STATS=${STATS:-dexhand_midtrain_m3} HF="$SYT/code/XPolicyLab/policy/Pi_05/openpi/.venv/bin/hf" cd "$SYT/lerobot" pats=(); for b in $BATCHES; do pats+=("lerobot/${b}_*seg_*/*"); done # cover_m2 shards are *_7seg_* (7-segment cut); ONE --include with all patterns: repeated for try in 1 2 3 4 5 6; do # --include flags keep only the last one if "$HF" download Noctis12138/dexhand-midtrain --repo-type dataset --include "${pats[@]}" --local-dir _hf_midtrain \ >> "$SYT/logs/fetch_midtrain.log" 2>&1; then break fi echo "download attempt $try failed, retrying in 60 s"; sleep 60 done mkdir -p dexhand for d in _hf_midtrain/lerobot/*seg_*/; do n=$(basename "$d"); [ -e "dexhand/$n" ] || ln -s "$PWD/$d" "dexhand/$n"; done echo "shards per batch:"; for b in $BATCHES; do echo " $b $(ls -d dexhand/${b}_*seg_* 2>/dev/null | wc -l)"; done echo "segments: $(python3 -c " import glob, json bs='$BATCHES'.split() print(sum(json.load(open(f))['total_episodes'] for b in bs for f in glob.glob(f'dexhand/{b}_*seg_*/meta/info.json')))")" if [ "${1:-}" = "--stats" ]; then cd "$SYT/code/XPolicyLab/policy/Pi_05/openpi" pat=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); pat=${pat%+} .venv/bin/python "$SYT/code/dexhand/train/run.py" norm-stats-parquet --repo-id "$pat" --stats "$STATS" fi