Upload scripts/ruler_suite.sh with huggingface_hub

scripts/ruler_suite.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# =============================================================================
+# FULL RULER SUITE on t64 -- the first real RULER-coverage number this project
+# will have. 13 tasks across RULER's four categories:
+#   retrieval      niah_single, _1, _3, multikey, _2, _3, multivalue, multiquery
+#   multi-hop      vt
+#   aggregation    cwe, fwe
+#   QA             qa_1, qa_2
+#
+# WHY THIS MATTERS: t64 scores 98-100 on niah_single at every length. A "RULER
+# score" built from that alone reads ~99 and means nothing -- RULER exists
+# precisely because vanilla NIAH saturates. multikey already went 39 -> 32 with
+# 64k training, so the suite is where the real number lives.
+#
+# Config: x64 at 32,744 -- t64's best measured setting (99 on niah_single).
+# t64 first (the model we care about), p1final second as control.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+L=ruler_logs; mkdir -p "$L"; M="$L/master.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+EXE=./build/bluscriptCP_exec
+HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+CK=checkpoints_bluscriptcp/rank_0
+[ -x "$EXE" ] || die "binary missing"
+[ "$(stat -c%s "$HAY")" = 199754694 ] || die "haystack differs from prior evals"
+[ -s "$CK/t64.ckpt" ] || die "t64 missing"
+A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+T=131072; S=32; PL=32744        # x32 -> virtual 1,023; t64 scores 99 on niah_single here
+[ $((S*4096)) -eq $T ] || die "YaRN invariant"
+
+# wait for the val_backward diagnostic to release the GPU
+while pgrep -x bluscriptCP_exec >/dev/null 2>&1; do sleep 30; done
+say "GPU free -- starting RULER suite"
+
+TASKS_ALL="niah_single niah_single_1 niah_single_3 niah_multikey niah_multikey_2 niah_multikey_3 \
+niah_multivalue niah_multiquery vt cwe fwe qa_1 qa_2"
+
+one(){ # model ckptfile
+  local NAME=$1 F=$2
+  for t in $TASKS_ALL; do
+    local O="ruler_${NAME}_${t}"
+    [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && { say "  $NAME/$t cached"; continue; }
+    say "  $NAME / $t"
+    A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
+    CKPT_EXTRA="$NAME=$F" CKPTS="$NAME" TASKS="$t" N=100 \
+    T=$T YARN=$S YARN_ORIG=4096 PROMPT_LEN=$PL GPUS=0 NP=1 OUT="$O" \
+      ./run_longeval.sh >> "$L/${NAME}_${t}.log" 2>&1
+    if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+      say "    $(tail -n +2 $O/summary.csv | tr '\n' ' ')"
+    else
+      say "    FAILED"; grep -ahA2 'what()\|Error\|Traceback' "$O"/gen/*.log "$L/${NAME}_${t}.log" 2>/dev/null|head -3|tee -a "$M" >&2
+    fi
+  done
+}
+say "=== RULER suite, 13 tasks, N=100, PL=$PL, T=$T x$S ==="
+one t64 t64.ckpt
+one p1final p1final.ckpt
+say "=== RULER TABLE ==="
+for t in $TASKS_ALL; do
+  a=$(tail -n +2 ruler_t64_${t}/summary.csv 2>/dev/null | cut -d, -f2 | head -1)
+  b=$(tail -n +2 ruler_p1final_${t}/summary.csv 2>/dev/null | cut -d, -f2 | head -1)
+  printf "  %-18s t64=%-6s p1final=%-6s\n" "$t" "${a:-.}" "${b:-.}" | tee -a "$M"
+done
+say "RULER DONE"