File size: 4,771 Bytes
32e6514 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | #!/usr/bin/env bash
# =============================================================================
# One command: 8x H200, 128k extension + NIAH eval. Setup ~17 min (build and
# downloads run CONCURRENTLY -- the build is the only critical path).
#
# WHY H200: 128k at cp=8 needs 16,384 tok/rank -> ~77,400 MB. Measured OOM on
# H100 (81,559 MB, 94.9%); on H200 (143,771 MB) that is 53.8%. cp=8 is the
# ceiling on 8 GPUs so there is no other lever.
#
# INVARIANT: SCALE 32 x ORIG 4096 == CP_T 131072. The 32k run broke this
# (32 x 4096 != 32768), trained on virtual 0..1023 instead of 0..4095, and went
# 90 -> 49. t128_train.sh hard-fails rather than repeat it.
# =============================================================================
set -uo pipefail
cd "$(dirname "$0")"
R=/root/Adhi/BluTrain/dist/Context_Parallelism
say(){ echo "[$(date -u +%H:%M:%S)] $*"; }
say "=== STAGE 1: build + downloads, concurrently ==="
( cd "$R"
make libtensor SM_ARCH=90 -j$(nproc) > b_lib.log 2>&1 || { echo LIBFAIL; exit 1; }
make CP_FUSED_ROPE=1 CP_ATTN_SPLIT=1 SM_ARCH=90 bluscript-cp -j$(nproc) > b_cp.log 2>&1
[ -x build/bluscriptCP_exec ] && echo "BUILD_OK $(stat -c%s build/bluscriptCP_exec)" || { echo BUILDFAIL; tail -20 b_cp.log; }
) > "$R/_build.out" 2>&1 &
BPID=$!
( cd "$R"
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
pip install -q tiktoken pyarrow 2>&1 | tail -1
mkdir -p checkpoints_680m/rank_0 checkpoints_bluscriptcp/rank_0 \
Data_Loader/Data/b2_8192_16384 Data_Loader/Data/evalsrc Tests/bluscriptcp/qa_data
R1=unparallelled/BluScriptCP; RD=unparallelled/BluScriptCP-data
g(){ hf download "$1" ${3:+--repo-type dataset} --quiet --include "$2" --local-dir "$4"; }
g $R1 'checkpoints/t64_680m_step1200_rank0.ckpt' "" dl1 && mv -f dl1/checkpoints/*.ckpt checkpoints_bluscriptcp/rank_0/t64.ckpt
g $R1 'checkpoints/p1_680m_step1200_rank0.ckpt' "" dl2 && mv -f dl2/checkpoints/*.ckpt checkpoints_bluscriptcp/rank_0/p1final.ckpt
hf download navingv/varybatch --quiet --include 'varybatch.ckpt' --local-dir dl3 && mv -f dl3/varybatch.ckpt checkpoints_680m/rank_0/
ln -sf "$PWD/checkpoints_680m/rank_0/varybatch.ckpt" checkpoints_bluscriptcp/rank_0/varybatch.ckpt
g $RD 'cp_data_by_length/b2_8192_16384/val_shard_00000.bin' 1 dl4 && mv -f dl4/cp_data_by_length/b2_8192_16384/*.bin Data_Loader/Data/b2_8192_16384/
for b in "b6_gte_131072 0 0" "b5_65536_131072 4 8" "b4_32768_65536 2 5" "b3_16384_32768 1 3" "b1_4096_8192 1 2"; do
set -- $b; for i in $(seq -f '%05g' $2 $3); do
hf download $RD --repo-type dataset --quiet --include "cp_data_by_length/$1/train_shard_$i.bin" --local-dir dl5 || true; done; done
for d in dl5/cp_data_by_length/*; do n=$(basename $d); mkdir -p src128/$n; mv -n $d/*.bin src128/$n/ 2>/dev/null; done
cp -f src128/b1_4096_8192/train_shard_*.bin Data_Loader/Data/evalsrc/ 2>/dev/null | head -1
cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/evalsrc/
g $R1 'data/ruler_qa/*' "" dl6; mv -f dl6/data/ruler_qa/*.parquet Tests/bluscriptcp/qa_data/ 2>/dev/null
g $R1 'scripts/longeval_gen.py' "" dl7 && cp -f dl7/scripts/longeval_gen.py Tests/bluscriptcp/
for s in t128_train.sh run_ruler.sh mix_buckets.py run_longeval.sh; do
g $R1 "scripts/$s" "" dl8 && cp -f dl8/scripts/$s . 2>/dev/null; done
chmod +x *.sh 2>/dev/null
rm -rf dl1 dl2 dl3 dl4 dl5 dl6 dl7 dl8
echo "DL_OK"
) > "$R/_dl.out" 2>&1 &
DPID=$!
wait $DPID; say "downloads: $(tail -1 $R/_dl.out)"
wait $BPID; say "build: $(tail -1 $R/_build.out)"
grep -q BUILD_OK "$R/_build.out" || { say "FATAL build failed"; tail -20 "$R/_build.out"; exit 1; }
grep -q DL_OK "$R/_dl.out" || { say "FATAL downloads failed"; tail -20 "$R/_dl.out"; exit 1; }
say "=== STAGE 2: mix (b6 8.9 / b5 39 / b4 25 / b3 15 / b1 10) ==="
cd "$R"; export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
python mix_buckets.py --out Data_Loader/Data/mix128 --data-root src128 \
--src b6_gte_131072=0.11 --src b5_65536_131072=0.39 --src b4_32768_65536=0.25 \
--src b3_16384_32768=0.15 --src b1_4096_8192=0.10 \
--total-tokens 640e6 --shard-tokens 96e6 --window 131072 --grad-accum 4 2>&1 | tail -8
cp -f Data_Loader/Data/b2_8192_16384/val_shard_00000.bin Data_Loader/Data/mix128/
say "=== STAGE 3: training (probe is the go/no-go) ==="
./t128_train.sh || { say "training stopped -- see t128_logs/master.log"; exit 1; }
say "=== STAGE 4: NIAH eval, 3 models x 6 lengths, all 8 GPUs ==="
RN=$(cat t128_logs/runnum); FIN=$(ls -t checkpoints_bluscriptcp/rank_0/blumodelcp_run${RN}_step_*.ckpt | head -1)
EXTRA="t128=$(basename $FIN) t64=t64.ckpt base680=varybatch.ckpt" \
MODELS="t128 t64 base680" LENS="4096 8192 16384 32744 65512 131000" N=100 \
RULER_TASKS="niah_single" ./run_ruler.sh
say "=== DONE ==="
|