File size: 1,692 Bytes
9791eca | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | #!/usr/bin/env bash
# Can 2x H100 TRAIN at 32k / 64k? Measure, do not extrapolate.
# Phase 1 measured 50,117 MB at T=16,384 cp=2. Everything above that has been a
# 2-point fit whose second point I can no longer verify.
set -uo pipefail
cd "$(dirname "$0")"
ROOT="$(cd ../.. && pwd)"
export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
export PATH=/usr/local/cuda/bin:$PATH
EXE=./build/bluscriptCP_exec
ARCH="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
CK=$PWD/checkpoints_680m/rank_0/varybatch.ckpt
probe(){ # T SCALE
local T=$1 S=$2
local L="memprobe_${T}.log"
echo "=== probe T=$T scale=$S cp=2 ($((T/2)) tok/rank) ==="
env $ARCH CP_SIZE=2 CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
YARN_SCALE=$S YARN_ORIG_MAXPOS=4096 CP_ROPE_THETA=500000 \
CP_ZERO=1 CP_ZERO_V2=1 CP_BF16=1 CP_OPTIMIZER=muon \
CP_WARMUP=2 CP_STABLE=0 CP_CKPT=0 CP_INIT_FROM="$CK" \
CP_DATA_ROOT="$PWD/Data_Loader/Data/evalsrc" \
CP_B=1 CP_GLOBAL_BATCH=$T CP_MAX_STEPS=3 \
CUDA_VISIBLE_DEVICES=0,1 mpirun --allow-run-as-root -np 2 "$EXE" > "$L" 2>&1
local rc=$?
if grep -qaiE 'out of memory|bad_alloc' "$L"; then
echo " OOM -> does NOT fit on 2x H100"
elif [ $rc -eq 0 ] && grep -qaE '^step +[0-9]+ \|' "$L"; then
echo " FITS: peak $(grep -aoE 'mem: [0-9]+MB' "$L" | tail -1) of 81559 MiB/card"
echo " step time $(grep -aoE 'dt: [0-9.]+ms' "$L" | tail -1)"
else
echo " FAILED rc=$rc"; grep -aA1 'terminate called' "$L" | grep -a 'what()' | head -1
fi
}
probe 32768 8
probe 65536 16
|