Upload scripts/run_ruler.sh with huggingface_hub

scripts/run_ruler.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Full RULER (13 tasks, 4 categories) on the 680M long-context models.
+#
+# PARALLELISM: run_longeval.sh fans out over (checkpoint x task) pairs. Passing
+# ONE task at a time gives 3 jobs for 8 GPUs -- which is what happened on
+# 2026-09-17 and left 5 cards idle per length. Passing ALL 13 tasks in a single
+# invocation gives 13 x models jobs per length and fills the box.
+#
+# RULER's 13:
+#   retrieval      niah_single{,_1,_3} niah_multikey{,_2,_3} multivalue multiquery
+#   multi-hop      vt
+#   aggregation    cwe fwe
+#   QA             qa_1 (SQuAD) qa_2 (HotpotQA)
+#
+# N: RULER publishes at 500/task. We use 100 by default for internal
+# comparability with every prior result in this project; set N=500 for numbers
+# comparable to the published RULER leaderboard (5x the time).
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+
+RULER="niah_single niah_single_1 niah_single_3 niah_multikey niah_multikey_2 \
+niah_multikey_3 niah_multivalue niah_multiquery vt cwe fwe qa_1 qa_2"
+LENS=${LENS:-"4096 8192 16384 32744 65512"}
+N=${N:-100}
+GPUS=${GPUS:-0,1,2,3,4,5,6,7}
+A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+HAY=${HAY:-Data_Loader/Data/b2_8192_16384/val_shard_00000.bin}
+CK=checkpoints_bluscriptcp/rank_0
+EXTRA=${EXTRA:-"t64=t64.ckpt p1final=p1final.ckpt base680=varybatch.ckpt"}
+MODELS=${MODELS:-"t64 p1final base680"}
+L=ruler_logs; mkdir -p "$L"; M="$L/master.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+
+[ -x ./build/bluscriptCP_exec ] || die "binary not built"
+grep -q 'CP_FUSED_ROPE=1' build/objects/.cp_rope_flags && \
+grep -q 'CP_ATTN_SPLIT=1' build/objects/.cp_rope_flags || die "binary lacks split kernels"
+[ -f "$HAY" ] || die "haystack missing: $HAY"
+[ "$(stat -c%s "$HAY")" = 199754694 ] || die "haystack is NOT the file every prior eval used"
+for f in Tests/bluscriptcp/qa_data/squad_val.parquet Tests/bluscriptcp/qa_data/hotpot_val.parquet; do
+  [ -f "$f" ] || die "QA data missing: $f  (qa_1/qa_2 cannot run)"; done
+python -c "import pyarrow" 2>/dev/null || die "pyarrow missing (needed by qa_1/qa_2)"
+for m in $MODELS; do
+  f=$(echo "$EXTRA" | tr ' ' '\n' | grep "^$m=" | cut -d= -f2)
+  [ -f "$CK/$f" ] || die "checkpoint for '$m' missing: $CK/$f"
+done
+
+say "=== FULL RULER: 13 tasks x $(echo $MODELS|wc -w) models x $(echo $LENS|wc -w) lengths, N=$N ==="
+say "models: $MODELS"
+say "jobs per length: $(( 13 * $(echo $MODELS|wc -w) ))  across $(echo $GPUS|tr ',' ' '|wc -w) GPUs"
+
+for PL in $LENS; do
+  O="ruler_len${PL}"
+  if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+    say "len $PL cached ($(( $(wc -l < "$O/summary.csv") - 1 )) task rows)"; continue; fi
+  # YaRN: scale must land content near virtual 1,000-2,000. SCALE=64/ORIG=4096
+  # (T=262,144) is the best config measured on these models; it beats the
+  # as-trained config by +57pp at 65,512.
+  say "len $PL  (T=262144 x64 -> virtual $((PL/64)))"
+  A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="${GEN_DATA_ROOT:-Data_Loader/Data/evalsrc}" \
+  CKPT_EXTRA="$EXTRA" CKPTS="$MODELS" TASKS="$RULER" N=$N \
+  T=262144 YARN=64 YARN_ORIG=4096 PROMPT_LEN=$PL GPUS="$GPUS" NP=1 OUT="$O" \
+    ./run_longeval.sh >> "$L/len_${PL}.log" 2>&1
+  if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+    say "  done: $(( $(wc -l < "$O/summary.csv") - 1 ))/13 tasks scored"
+    tail -n +2 "$O/summary.csv" | sed 's/^/    /' | tee -a "$M" >&2
+  else
+    say "  FAILED at len $PL"; grep -ahA2 'what()' "$O"/gen/*.log 2>/dev/null | head -4 | tee -a "$M" >&2
+  fi
+done
+say "=== RULER COMPLETE ==="
+for PL in $LENS; do
+  f="ruler_len${PL}/summary.csv"
+  [ -f "$f" ] && { say "len $PL:"; cat "$f" | sed 's/^/    /' | tee -a "$M" >&2; }
+done