File size: 1,274 Bytes
6a08a28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/usr/bin/env bash
# Receive a checkpoint relayed by train/ms_relay.sh (public ModelScope repo, no token): download the parts in parallel,
# check the part SHA-256s, untar, check MANIFEST.sha256 of every file, then delete the parts.
#   bash train/ms_fetch.sh <step> [parallel=16]     (on the 5090; output /data/syt/dexhand/checkpoints/<exp>/<step>)
set -euo pipefail
step=$1; par=${2:-16}
EXP=${EXP:-dexhand_cover_v2_ilv}
REPO=${REPO:-N0ct1s/dexhand-pi05-cover-ilv}
TAG=${EXP}_$step
DST=/data/syt/dexhand/checkpoints/$EXP
TMP=/data/syt/dexhand/tmp/fetch_$TAG
URL="https://www.modelscope.cn/models/$REPO/resolve/master/relay/$TAG"
mkdir -p "$DST" "$TMP"; cd "$TMP"
curl -fsSL --retry 5 -o "$TAG.parts.sha256" "$URL/$TAG.parts.sha256"
curl -fsSL --retry 5 -o MANIFEST.sha256 "$URL/MANIFEST.sha256"
awk '{print $2}' "$TAG.parts.sha256" | xargs -P "$par" -I{} curl -fsSL --retry 5 -C - -o {} "$URL/{}"
sha256sum -c --quiet "$TAG.parts.sha256"
cat "$TAG".tar.part-* | tar -xf - -C "$DST"
(cd "$DST" && sha256sum -c --quiet "$TMP/MANIFEST.sha256")
echo "verified $(wc -l < MANIFEST.sha256) files -> $DST/$step"
cd /; rm -rf "$TMP"
echo "- $(date -u +%H:%MZ) fetched $TAG from ModelScope $REPO -> $DST/$step (parts + MANIFEST sha256 verified)" >> /data/syt/dexhand/PROVENANCE.md