#!/usr/bin/env bash # Receive a checkpoint relayed by train/ms_relay.sh (public ModelScope repo, no token): download the parts in parallel, # check the part SHA-256s, untar, check MANIFEST.sha256 of every file, then delete the parts. # bash train/ms_fetch.sh [parallel=16] (on the 5090; output /data/syt/dexhand/checkpoints//) set -euo pipefail step=$1; par=${2:-16} EXP=${EXP:-dexhand_cover_v2_ilv} REPO=${REPO:-N0ct1s/dexhand-pi05-cover-ilv} TAG=${EXP}_$step DST=/data/syt/dexhand/checkpoints/$EXP TMP=/data/syt/dexhand/tmp/fetch_$TAG URL="https://www.modelscope.cn/models/$REPO/resolve/master/relay/$TAG" mkdir -p "$DST" "$TMP"; cd "$TMP" curl -fsSL --retry 5 -o "$TAG.parts.sha256" "$URL/$TAG.parts.sha256" curl -fsSL --retry 5 -o MANIFEST.sha256 "$URL/MANIFEST.sha256" awk '{print $2}' "$TAG.parts.sha256" | xargs -P "$par" -I{} curl -fsSL --retry 5 -C - -o {} "$URL/{}" sha256sum -c --quiet "$TAG.parts.sha256" cat "$TAG".tar.part-* | tar -xf - -C "$DST" (cd "$DST" && sha256sum -c --quiet "$TMP/MANIFEST.sha256") echo "verified $(wc -l < MANIFEST.sha256) files -> $DST/$step" cd /; rm -rf "$TMP" echo "- $(date -u +%H:%MZ) fetched $TAG from ModelScope $REPO -> $DST/$step (parts + MANIFEST sha256 verified)" >> /data/syt/dexhand/PROVENANCE.md