| # Download ONLY the buckets phases 2-3 need. Explicit --include always: | |
| # an unfiltered `hf download` of this repo pulled 167 GB and broke the build. | |
| set -uo pipefail | |
| cd "$(dirname "$0")" | |
| export PATH=/root/Adhi/BluTrain/envv/bin:$PATH | |
| R=unparallelled/BluScriptCP-data | |
| say(){ echo "[$(date -u +%H:%M:%S)] $*"; } | |
| get(){ # bucketdir first last | |
| local b=$1 f=$2 l=$3 | |
| for i in $(seq -f '%05g' "$f" "$l"); do | |
| say " $b/train_shard_$i.bin" | |
| hf download "$R" --repo-type dataset --quiet \ | |
| --include "cp_data_by_length/$b/train_shard_$i.bin" --local-dir dl_tmp || return 1 | |
| done | |
| } | |
| say "=== b4 (phase 2 long + phase 3 filler): 13 shards, 2.54 GB ==="; get b4_32768_65536 0 12 || exit 1 | |
| say "=== b5 (phase 3 long): 10 shards, 1.92 GB ==="; get b5_65536_131072 0 9 || exit 1 | |
| say "=== b3 extra (fresh filler range): shards 4-8 ==="; get b3_16384_32768 4 8 || exit 1 | |
| say "=== b1 extra (fresh filler range): shards 2-6 ==="; get b1_4096_8192 2 6 || exit 1 | |
| for d in dl_tmp/cp_data_by_length/*; do | |
| [ -d "$d" ] || continue; n=$(basename "$d"); mkdir -p "src1/$n" | |
| for f in "$d"/*.bin; do [ -e "$f" ] || continue; mv -n "$f" "src1/$n/"; done | |
| done | |
| rm -rf dl_tmp | |
| say "=== DONE ==="; du -sh src1/*; df -h . | tail -1 | |