Upload scripts/dl_vb.sh with huggingface_hub

scripts/dl_vb.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Only what the val_backward diagnostic needs.
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
+mkdir -p checkpoints_680m/rank_0 checkpoints_bluscriptcp/rank_0 \
+         Data_Loader/Data/b2_8192_16384 Data_Loader/Data/evalsrc
+R=unparallelled/BluScriptCP; RD=unparallelled/BluScriptCP-data
+say "1/4 base680 (varybatch)"
+hf download navingv/varybatch --quiet --include 'varybatch.ckpt' --local-dir dlv || exit 1
+mv -f dlv/varybatch.ckpt checkpoints_680m/rank_0/; rm -rf dlv
+ln -sf "$PWD/checkpoints_680m/rank_0/varybatch.ckpt" checkpoints_bluscriptcp/rank_0/varybatch.ckpt
+say "2/4 p1final (best model: 95/90/82 at x64)"
+hf download $R --quiet --include 'checkpoints/p1_680m_step1200_rank0.ckpt' --local-dir dlp || exit 1
+mv -f dlp/checkpoints/p1_680m_step1200_rank0.ckpt checkpoints_bluscriptcp/rank_0/p1final.ckpt; rm -rf dlp
+say "3/4 haystack -- byte-identical to every prior eval"
+hf download $RD --repo-type dataset --quiet \
+  --include 'cp_data_by_length/b2_8192_16384/val_shard_00000.bin' --local-dir dlh || exit 1
+mv -f dlh/cp_data_by_length/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/b2_8192_16384/; rm -rf dlh
+say "4/4 loader feed (trainer builds a loader even for pure generation)"
+hf download $RD --repo-type dataset --quiet \
+  --include 'cp_data_by_length/b1_4096_8192/train_shard_00000.bin' --local-dir dlt || exit 1
+mv -f dlt/cp_data_by_length/b1_4096_8192/train_shard_00000.bin Data_Loader/Data/evalsrc/; rm -rf dlt
+cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/evalsrc/
+say "=== sizes ==="
+stat -c '  %s  %n' checkpoints_680m/rank_0/varybatch.ckpt \
+  checkpoints_bluscriptcp/rank_0/p1final.ckpt \
+  Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+say "expected: varybatch 1362674564 | p1final 5141244514 | haystack 199754694"
+say "DL_DONE"