Download code/dexhand/train/prepare_midtrain.sh from Noctis12138/dexhand-train-kit: direct link, hf CLI and curl.
- Browser
- Download file 2.32 kB
-
https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/dexhand/train/prepare_midtrain.sh
- Command line
-
hf download hf://Noctis12138/dexhand-train-kit/code/dexhand/train/prepare_midtrain.sh
-
curl -L -o prepare_midtrain.sh https://huggingface.co/Noctis12138/dexhand-train-kit/resolve/main/code/dexhand/train/prepare_midtrain.sh
2.32 kB
| # A100 (Autoresearch3): fetch the model-v2 mid-training shards (public HF dataset Noctis12138/dexhand-midtrain via | |
| # hf-mirror; re-run to pick up new ones) into $SYT/lerobot/dexhand/, then optionally compute the norm stats of the | |
| # 10-batch mixture. Batches (2026-10-24): cover_m2 align_v2 bottles_v2 blocks_v2 bowls_v2 pens_v2 pickup_v2 pickup_v3 | |
| # pack_v2 dolls_v2 - 3,569 sub-task segments, 555K frames, 54-D joints, 25 fps, model v2. | |
| # bash train/prepare_midtrain.sh fetch (resumable) | |
| # bash train/prepare_midtrain.sh --stats also compute norm stats -> $SYT/assets/dexhand_midtrain | |
| set -euo pipefail | |
| source "${SYT:-/share/project/zhiwei/syt}/env.sh" | |
| # mid-train v3 (2026-10-27); the v2 batches of mid-train v1 (cover_m2 align_v2 ... dolls_v2) were removed from HF that day | |
| BATCHES=${BATCHES:-"cover_m3 align_m3 bottles_m3 blocks_m3 bowls_m3 pens_m3 pickup_m3a pickup_m3b pack_m3 dolls_m3"} | |
| STATS=${STATS:-dexhand_midtrain_m3} | |
| HF="$SYT/code/XPolicyLab/policy/Pi_05/openpi/.venv/bin/hf" | |
| cd "$SYT/lerobot" | |
| pats=(); for b in $BATCHES; do pats+=("lerobot/${b}_*seg_*/*"); done # cover_m2 shards are *_7seg_* (7-segment cut); ONE --include with all patterns: repeated | |
| for try in 1 2 3 4 5 6; do # --include flags keep only the last one | |
| if "$HF" download Noctis12138/dexhand-midtrain --repo-type dataset --include "${pats[@]}" --local-dir _hf_midtrain \ | |
| >> "$SYT/logs/fetch_midtrain.log" 2>&1; then | |
| break | |
| fi | |
| echo "download attempt $try failed, retrying in 60 s"; sleep 60 | |
| done | |
| mkdir -p dexhand | |
| for d in _hf_midtrain/lerobot/*seg_*/; do n=$(basename "$d"); [ -e "dexhand/$n" ] || ln -s "$PWD/$d" "dexhand/$n"; done | |
| echo "shards per batch:"; for b in $BATCHES; do echo " $b $(ls -d dexhand/${b}_*seg_* 2>/dev/null | wc -l)"; done | |
| echo "segments: $(python3 -c " | |
| import glob, json | |
| bs='$BATCHES'.split() | |
| print(sum(json.load(open(f))['total_episodes'] for b in bs for f in glob.glob(f'dexhand/{b}_*seg_*/meta/info.json')))")" | |
| if [ "${1:-}" = "--stats" ]; then | |
| cd "$SYT/code/XPolicyLab/policy/Pi_05/openpi" | |
| pat=$(for b in $BATCHES; do printf "glob:dexhand/%s_*seg_*+" $b; done); pat=${pat%+} | |
| .venv/bin/python "$SYT/code/dexhand/train/run.py" norm-stats-parquet --repo-id "$pat" --stats "$STATS" | |
| fi | |