File size: 723 Bytes
ec9d3b4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | #!/usr/bin/env bash
set -uo pipefail; cd "$(dirname "$0")"
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
R=unparallelled/BluScriptCP-data
get(){ for i in $(seq -f '%05g' "$2" "$3"); do
hf download "$R" --repo-type dataset --quiet \
--include "cp_data_by_length/$1/train_shard_$i.bin" --local-dir dl32 || return 1; done; }
echo "b4 32-64k (docs >= the 32k window: contiguous single-doc context)"; get b4_32768_65536 0 6
echo "b3 16-32k"; get b3_16384_32768 0 3
echo "b1 4-8k"; get b1_4096_8192 0 2
for d in dl32/cp_data_by_length/*; do n=$(basename "$d"); mkdir -p "src32/$n"
for f in "$d"/*.bin; do [ -e "$f" ] && mv -n "$f" "src32/$n/"; done; done
rm -rf dl32; du -sh src32/*; df -h . | tail -1; echo DL_DONE
|