Upload scripts/prep128.sh with huggingface_hub

scripts/prep128.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# CPU/IO only -- safe to run against the GPU eval.
+# Builds a 128k mix from token ranges DISJOINT from what the 64k run consumed,
+# so this is not a second epoch over the same data.
+#   64k run used: b5 315M (~3.3 shards), b4 157M (~1.6), b3 94M (~1), b1 63M
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
+A=$(pwd)
+
+say "1/3 b6 (>=131k docs -- the only bucket exceeding a 128k window)"
+for i in 00000 00001; do
+  hf download unparallelled/BluScriptCP-data --repo-type dataset --quiet \
+    --include "cp_data_by_length/b6_gte_131072/train_shard_$i.bin" --local-dir dl6 || true
+done
+mkdir -p src128/b6_gte_131072
+find dl6 -name '*.bin' -exec mv -n {} src128/b6_gte_131072/ \; 2>/dev/null; rm -rf dl6
+say "  b6 shards: $(ls src128/b6_gte_131072 2>/dev/null | wc -l)  $(du -sh src128/b6_gte_131072 2>/dev/null | cut -f1)"
+
+say "2/3 symlink FRESH shard ranges from the buckets already on disk"
+link(){ mkdir -p "src128/$1"; for i in $(seq -f '%05g' "$2" "$3"); do
+  [ -f "$A/src64/$1/train_shard_$i.bin" ] && ln -sf "$A/src64/$1/train_shard_$i.bin" "src128/$1/train_shard_$i.bin"; done; }
+link b5_65536_131072 4 8      # 64k run ate 0-3
+link b4_32768_65536  2 5      # ate 0-1
+link b3_16384_32768  1 3      # ate 0
+link b1_4096_8192    1 2      # ate 0
+for d in src128/*/; do echo "    $(basename $d): $(ls $d | wc -l) shards"; done
+
+say "3/3 build the mix -- b6 11 / b5 39 / b4 25 / b3 15 / b1 10"
+python mix_buckets.py --out Data_Loader/Data/mix128 --data-root src128 \
+  --src b6_gte_131072=0.11 --src b5_65536_131072=0.39 --src b4_32768_65536=0.25 \
+  --src b3_16384_32768=0.15 --src b1_4096_8192=0.10 \
+  --total-tokens 640e6 --shard-tokens 96e6 --window 131072 --grad-accum 4 2>&1 | tail -14
+cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/mix128/ 2>/dev/null
+say "PREP_DONE"; df -h . | tail -1