Upload scripts/dl_64k.sh with huggingface_hub

scripts/dl_64k.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Everything the 64k extension needs. Explicit --include on every call.
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
+mkdir -p checkpoints_680m/rank_0 checkpoints_bluscriptcp/rank_0 \
+         Data_Loader/Data/b2_8192_16384 Data_Loader/Data/evalsrc
+R=unparallelled/BluScriptCP; RD=unparallelled/BluScriptCP-data
+
+say "1/5 p1final (the parent -- best model we have, 95/90/82 at x64)"
+hf download $R --quiet --include 'checkpoints/p1_680m_step1200_rank0.ckpt' --local-dir dlp || exit 1
+mv -f dlp/checkpoints/p1_680m_step1200_rank0.ckpt checkpoints_bluscriptcp/rank_0/p1final.ckpt; rm -rf dlp
+
+say "2/5 base680 (the control -- without it we cannot read the result)"
+hf download navingv/varybatch --quiet --include 'varybatch.ckpt' --local-dir dlv || exit 1
+mv -f dlv/varybatch.ckpt checkpoints_680m/rank_0/; rm -rf dlv
+ln -sf "$PWD/checkpoints_680m/rank_0/varybatch.ckpt" checkpoints_bluscriptcp/rank_0/varybatch.ckpt
+
+say "3/5 haystack -- MUST be byte-identical to every prior eval"
+hf download $RD --repo-type dataset --quiet \
+  --include 'cp_data_by_length/b2_8192_16384/val_shard_00000.bin' --local-dir dlh || exit 1
+mv -f dlh/cp_data_by_length/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/b2_8192_16384/; rm -rf dlh
+
+say "4/5 long buckets for a 64k window"
+get(){ for i in $(seq -f '%05g' "$2" "$3"); do
+  hf download $RD --repo-type dataset --quiet \
+    --include "cp_data_by_length/$1/train_shard_$i.bin" --local-dir dld || return 1; done; }
+get b5_65536_131072 0 8 || exit 1     # docs >= the 64k window: contiguous
+get b4_32768_65536  0 5 || exit 1
+get b3_16384_32768  0 3 || exit 1
+get b1_4096_8192    0 2 || exit 1
+for d in dld/cp_data_by_length/*; do n=$(basename "$d"); mkdir -p "src64/$n"
+  for f in "$d"/*.bin; do [ -e "$f" ] && mv -n "$f" "src64/$n/"; done; done
+rm -rf dld
+
+say "5/5 loader feed (trainer builds a data loader even for pure generation)"
+cp -f src64/b1_4096_8192/train_shard_00000.bin Data_Loader/Data/evalsrc/ 2>/dev/null
+cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/evalsrc/
+
+say "=== sizes ==="
+stat -c '  %s  %n' checkpoints_bluscriptcp/rank_0/p1final.ckpt \
+  checkpoints_680m/rank_0/varybatch.ckpt \
+  Data_Loader/Data/b2_8192_16384/val_shard_00000.bin 2>/dev/null
+say "expected: p1final 5141244514 | varybatch 1362674564 | haystack 199754694"
+du -sh src64/* ; df -h . | tail -1
+say "DL_DONE"