File size: 1,898 Bytes
ba6a188 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | #!/usr/bin/env bash
# Only what the val_backward diagnostic needs.
set -uo pipefail; cd "$(dirname "$0")"
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
mkdir -p checkpoints_680m/rank_0 checkpoints_bluscriptcp/rank_0 \
Data_Loader/Data/b2_8192_16384 Data_Loader/Data/evalsrc
R=unparallelled/BluScriptCP; RD=unparallelled/BluScriptCP-data
say "1/4 base680 (varybatch)"
hf download navingv/varybatch --quiet --include 'varybatch.ckpt' --local-dir dlv || exit 1
mv -f dlv/varybatch.ckpt checkpoints_680m/rank_0/; rm -rf dlv
ln -sf "$PWD/checkpoints_680m/rank_0/varybatch.ckpt" checkpoints_bluscriptcp/rank_0/varybatch.ckpt
say "2/4 p1final (best model: 95/90/82 at x64)"
hf download $R --quiet --include 'checkpoints/p1_680m_step1200_rank0.ckpt' --local-dir dlp || exit 1
mv -f dlp/checkpoints/p1_680m_step1200_rank0.ckpt checkpoints_bluscriptcp/rank_0/p1final.ckpt; rm -rf dlp
say "3/4 haystack -- byte-identical to every prior eval"
hf download $RD --repo-type dataset --quiet \
--include 'cp_data_by_length/b2_8192_16384/val_shard_00000.bin' --local-dir dlh || exit 1
mv -f dlh/cp_data_by_length/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/b2_8192_16384/; rm -rf dlh
say "4/4 loader feed (trainer builds a loader even for pure generation)"
hf download $RD --repo-type dataset --quiet \
--include 'cp_data_by_length/b1_4096_8192/train_shard_00000.bin' --local-dir dlt || exit 1
mv -f dlt/cp_data_by_length/b1_4096_8192/train_shard_00000.bin Data_Loader/Data/evalsrc/; rm -rf dlt
cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/evalsrc/
say "=== sizes ==="
stat -c ' %s %n' checkpoints_680m/rank_0/varybatch.ckpt \
checkpoints_bluscriptcp/rank_0/p1final.ckpt \
Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
say "expected: varybatch 1362674564 | p1final 5141244514 | haystack 199754694"
say "DL_DONE"
|