Upload scripts/dl128.sh with huggingface_hub

scripts/dl128.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+# CPU/IO only. Safe to run while someone else has the GPUs.
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
+R1=unparallelled/BluScriptCP; RD=unparallelled/BluScriptCP-data
+mkdir -p checkpoints_680m/rank_0 checkpoints_bluscriptcp/rank_0 \
+         Data_Loader/Data/b2_8192_16384 Data_Loader/Data/evalsrc Tests/bluscriptcp/qa_data
+
+say "1/6 checkpoints: t64 (parent) + p1final + base680 (eval controls)"
+hf download $R1 --quiet --include 'checkpoints/t64_680m_step1200_rank0.ckpt' --local-dir d1 && \
+  mv -f d1/checkpoints/*.ckpt checkpoints_bluscriptcp/rank_0/t64.ckpt
+hf download $R1 --quiet --include 'checkpoints/p1_680m_step1200_rank0.ckpt' --local-dir d2 && \
+  mv -f d2/checkpoints/*.ckpt checkpoints_bluscriptcp/rank_0/p1final.ckpt
+hf download navingv/varybatch --quiet --include 'varybatch.ckpt' --local-dir d3 && \
+  mv -f d3/varybatch.ckpt checkpoints_680m/rank_0/
+ln -sf "$PWD/checkpoints_680m/rank_0/varybatch.ckpt" checkpoints_bluscriptcp/rank_0/varybatch.ckpt
+
+say "2/6 haystack -- MUST be byte-identical to every prior eval"
+hf download $RD --repo-type dataset --quiet \
+  --include 'cp_data_by_length/b2_8192_16384/val_shard_00000.bin' --local-dir d4 && \
+  mv -f d4/cp_data_by_length/b2_8192_16384/*.bin Data_Loader/Data/b2_8192_16384/
+
+say "3/6 buckets -- FRESH shard ranges, disjoint from what the 64k run consumed"
+for spec in "b6_gte_131072 0 0" "b5_65536_131072 4 8" "b4_32768_65536 2 5" "b3_16384_32768 1 3" "b1_4096_8192 1 2"; do
+  set -- $spec
+  for i in $(seq -f '%05g' $2 $3); do
+    hf download $RD --repo-type dataset --quiet \
+      --include "cp_data_by_length/$1/train_shard_$i.bin" --local-dir d5 || true
+  done
+done
+for d in d5/cp_data_by_length/*; do n=$(basename "$d"); mkdir -p "src128/$n"
+  for f in "$d"/*.bin; do [ -e "$f" ] && mv -n "$f" "src128/$n/"; done; done
+
+say "4/6 loader feed (trainer builds a loader even for pure generation)"
+cp -f src128/b1_4096_8192/train_shard_00001.bin Data_Loader/Data/evalsrc/ 2>/dev/null
+cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/evalsrc/
+
+say "5/6 scripts + RULER QA data"
+hf download $R1 --quiet --include 'data/ruler_qa/*' --local-dir d6 && \
+  mv -f d6/data/ruler_qa/*.parquet Tests/bluscriptcp/qa_data/
+hf download $R1 --quiet --include 'scripts/longeval_gen.py' --local-dir d7 && \
+  cp -f d7/scripts/longeval_gen.py Tests/bluscriptcp/
+for s in t128_train.sh run_ruler.sh mix_buckets.py run_longeval.sh longeval_score.py; do
+  hf download $R1 --quiet --include "scripts/$s" --local-dir d8 2>/dev/null || true; done
+cp -f d8/scripts/t128_train.sh d8/scripts/run_ruler.sh d8/scripts/mix_buckets.py d8/scripts/run_longeval.sh . 2>/dev/null
+cp -f d8/scripts/longeval_score.py Tests/bluscriptcp/ 2>/dev/null
+chmod +x *.sh 2>/dev/null
+rm -rf d1 d2 d3 d4 d5 d6 d7 d8
+
+say "6/6 sizes"
+stat -c '  %s  %n' checkpoints_bluscriptcp/rank_0/t64.ckpt \
+  checkpoints_bluscriptcp/rank_0/p1final.ckpt \
+  checkpoints_680m/rank_0/varybatch.ckpt \
+  Data_Loader/Data/b2_8192_16384/val_shard_00000.bin 2>/dev/null
+say "expect: t64 2423335522 | p1final 5141244514 | varybatch 1362674564 | haystack 199754694"
+du -sh src128/* 2>/dev/null; df -h . | tail -1
+say "DL_DONE"