Upload scripts/dl_t32.sh with huggingface_hub

scripts/dl_t32.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+R=unparallelled/BluScriptCP-data
+get(){ for i in $(seq -f '%05g' "$2" "$3"); do
+  hf download "$R" --repo-type dataset --quiet \
+    --include "cp_data_by_length/$1/train_shard_$i.bin" --local-dir dl32 || return 1; done; }
+echo "b4 32-64k (docs >= the 32k window: contiguous single-doc context)"; get b4_32768_65536 0 6
+echo "b3 16-32k"; get b3_16384_32768 0 3
+echo "b1 4-8k";   get b1_4096_8192   0 2
+for d in dl32/cp_data_by_length/*; do n=$(basename "$d"); mkdir -p "src32/$n"
+  for f in "$d"/*.bin; do [ -e "$f" ] && mv -n "$f" "src32/$n/"; done; done
+rm -rf dl32; du -sh src32/*; df -h . | tail -1; echo DL_DONE