| set -uo pipefail; cd "$(dirname "$0")" | |
| export PATH=/root/Adhi/BluTrain/envv/bin:$PATH | |
| R=unparallelled/BluScriptCP-data | |
| get(){ for i in $(seq -f '%05g' "$2" "$3"); do | |
| hf download "$R" --repo-type dataset --quiet \ | |
| --include "cp_data_by_length/$1/train_shard_$i.bin" --local-dir dl32 || return 1; done; } | |
| echo "b4 32-64k (docs >= the 32k window: contiguous single-doc context)"; get b4_32768_65536 0 6 | |
| echo "b3 16-32k"; get b3_16384_32768 0 3 | |
| echo "b1 4-8k"; get b1_4096_8192 0 2 | |
| for d in dl32/cp_data_by_length/*; do n=$(basename "$d"); mkdir -p "src32/$n" | |
| for f in "$d"/*.bin; do [ -e "$f" ] && mv -n "$f" "src32/$n/"; done; done | |
| rm -rf dl32; du -sh src32/*; df -h . | tail -1; echo DL_DONE | |