Upload scripts/stop500.sh with huggingface_hub

scripts/stop500.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Stop AT step 500 (all 8 ranks saved), then eval 4 lengths at x32 on 8 GPUs.
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+L=t128b_logs; M="$L/master.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+say "stop-watcher armed: waiting for step-500 save on all 8 ranks"
+while true; do
+  RN=$(cat $L/runnum 2>/dev/null || ls checkpoints_bluscriptcp/rank_0/ 2>/dev/null | grep -oE 'run[0-9]+' | sort -u | tail -1 | tr -d 'run')
+  n=$(ls checkpoints_bluscriptcp/rank_*/blumodelcp_run${RN}_step_500.ckpt 2>/dev/null|wc -l)
+  z=$(ls checkpoints_bluscriptcp/rank_*/blumodelcp_run${RN}_step_500.zero_g1 2>/dev/null|wc -l)
+  [ "$n" -ge 8 ] && [ "$z" -ge 8 ] && break
+  ps -eo args --no-headers | grep -q '[b]luscriptCP_exec' || { say "trainer gone early (n=$n)"; break; }
+  sleep 20
+done
+sleep 15
+say "step 500 saved on all ranks -- stopping trainer"
+for p in $(ps -eo pid,args --no-headers | grep -E '[b]luscriptCP_exec|[m]pirun' | awk '{print $1}'); do kill -9 "$p" 2>/dev/null; done
+sleep 6
+say "STOPPED AT 500  (exec left: $(ps -eo args --no-headers|grep -c '[b]luscriptCP_exec'))"
+RN=$(cat $L/runnum 2>/dev/null || ls checkpoints_bluscriptcp/rank_0/ | grep -oE 'run[0-9]+' | sort -u | tail -1 | tr -d 'run')
+ln -sf "$PWD/checkpoints_bluscriptcp/rank_0/blumodelcp_run${RN}_step_500.ckpt" \
+       checkpoints_bluscriptcp/rank_0/t128b_s500.ckpt
+A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+blk(){ local PL=$1 T=$2 G=$3 O="b500_$1"
+  A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
+  CKPT_EXTRA="s500=t128b_s500.ckpt" CKPTS="s500" TASKS="niah_single" N=100 \
+  T=$T YARN=32 YARN_ORIG=4096 PROMPT_LEN=$PL GPUS="$G" NP=1 OUT="$O" \
+    ./run_longeval.sh > "$L/ev_${PL}.log" 2>&1
+  say "  len $PL: $(grep -h '^niah_single,' $O/summary.csv 2>/dev/null|cut -d, -f2)"; }
+say "EVAL: 4 lengths at x32, one GPU each"
+blk 65512 65536 0 & blk 32744 32768 1 & blk 16360 16384 2 & blk 4096 8192 3 &
+wait
+say "=== t128b step500 @x32  vs  t64 @x32  vs  t128(old) @x32 ==="
+for c in 4096:100:98 16360:99:98 32744:93:69 65512:63:31; do
+  PL=${c%%:*}; r=${c#*:}; t64=${r%%:*}; old=${r##*:}
+  v=$(grep -h '^niah_single,' b500_${PL}/summary.csv 2>/dev/null|cut -d, -f2)
+  printf "  %-7s new=%-6s t64=%-5s old128=%s\n" "$PL" "${v:-.}" "$t64" "$old" | tee -a "$M"
+done
+say "EVAL DONE"