File size: 2,144 Bytes
856d2d1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env bash
# Pull exactly what the 32k eval needs. Explicit --include on every call:
# an unfiltered download of this repo once pulled 167 GB and broke a build.
set -uo pipefail
cd "$(dirname "$0")"
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
mkdir -p checkpoints_680m/rank_0 checkpoints_bluscriptcp/rank_0 Data_Loader/Data/b2_8192_16384 Data_Loader/Data/evalsrc
say "1/4 varybatch.ckpt (the untrained base -- the control)"
hf download navingv/varybatch --quiet --include 'varybatch.ckpt' --local-dir dl_vb || exit 1
mv -f dl_vb/varybatch.ckpt checkpoints_680m/rank_0/ && rm -rf dl_vb
say "2/4 p1final step1200 rank0 (5.14 GB)"
hf download unparallelled/BluScriptCP --quiet --include 'checkpoints/p1_680m_step1200_rank0.ckpt' --local-dir dl_p1 || exit 1
mv -f dl_p1/checkpoints/p1_680m_step1200_rank0.ckpt \
checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_1200.ckpt && rm -rf dl_p1
say "3/4 haystack -- MUST be the same file phase 1 used or the numbers are not comparable"
hf download unparallelled/BluScriptCP-data --repo-type dataset --quiet \
--include 'cp_data_by_length/b2_8192_16384/val_shard_00000.bin' --local-dir dl_hay || exit 1
mv -f dl_hay/cp_data_by_length/b2_8192_16384/val_shard_00000.bin \
Data_Loader/Data/b2_8192_16384/ && rm -rf dl_hay
say "4/4 a train split for the loader -- the trainer builds one even for pure generation"
hf download unparallelled/BluScriptCP-data --repo-type dataset --quiet \
--include 'cp_data_by_length/b1_4096_8192/train_shard_00000.bin' --local-dir dl_tr || exit 1
mv -f dl_tr/cp_data_by_length/b1_4096_8192/train_shard_00000.bin Data_Loader/Data/evalsrc/ && rm -rf dl_tr
cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/evalsrc/
say "=== sizes ==="
stat -c ' %s %n' checkpoints_680m/rank_0/varybatch.ckpt \
checkpoints_bluscriptcp/rank_0/blumodelcp_run12_step_1200.ckpt \
Data_Loader/Data/b2_8192_16384/val_shard_00000.bin \
Data_Loader/Data/evalsrc/train_shard_00000.bin 2>/dev/null
say "expected: varybatch 1362674564 | p1final 5141244514 | haystack 199754694"
say "DOWNLOAD DONE"
|