Upload scripts/memprobe.sh with huggingface_hub

scripts/memprobe.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Can 2x H100 TRAIN at 32k / 64k? Measure, do not extrapolate.
+# Phase 1 measured 50,117 MB at T=16,384 cp=2. Everything above that has been a
+# 2-point fit whose second point I can no longer verify.
+set -uo pipefail
+cd "$(dirname "$0")"
+ROOT="$(cd ../.. && pwd)"
+export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
+export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+export PATH=/usr/local/cuda/bin:$PATH
+EXE=./build/bluscriptCP_exec
+ARCH="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+CK=$PWD/checkpoints_680m/rank_0/varybatch.ckpt
+probe(){ # T SCALE
+  local T=$1 S=$2
+  local L="memprobe_${T}.log"
+  echo "=== probe T=$T scale=$S cp=2 ($((T/2)) tok/rank) ==="
+  env $ARCH CP_SIZE=2 CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
+      YARN_SCALE=$S YARN_ORIG_MAXPOS=4096 CP_ROPE_THETA=500000 \
+      CP_ZERO=1 CP_ZERO_V2=1 CP_BF16=1 CP_OPTIMIZER=muon \
+      CP_WARMUP=2 CP_STABLE=0 CP_CKPT=0 CP_INIT_FROM="$CK" \
+      CP_DATA_ROOT="$PWD/Data_Loader/Data/evalsrc" \
+      CP_B=1 CP_GLOBAL_BATCH=$T CP_MAX_STEPS=3 \
+      CUDA_VISIBLE_DEVICES=0,1 mpirun --allow-run-as-root -np 2 "$EXE" > "$L" 2>&1
+  local rc=$?
+  if grep -qaiE 'out of memory|bad_alloc' "$L"; then
+    echo "  OOM -> does NOT fit on 2x H100"
+  elif [ $rc -eq 0 ] && grep -qaE '^step +[0-9]+ \|' "$L"; then
+    echo "  FITS: peak $(grep -aoE 'mem: [0-9]+MB' "$L" | tail -1) of 81559 MiB/card"
+    echo "        step time $(grep -aoE 'dt: [0-9.]+ms' "$L" | tail -1)"
+  else
+    echo "  FAILED rc=$rc"; grep -aA1 'terminate called' "$L" | grep -a 'what()' | head -1
+  fi
+}
+probe 32768 8
+probe 65536 16