Upload scripts/ruler_full.sh with huggingface_hub

scripts/ruler_full.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# =============================================================================
+# FULL RULER: 13 tasks x 5 lengths, t64 (+ optional controls).
+#
+# TWO OPTIMISATIONS over the first attempt:
+#  1. MINIMUM T PER LENGTH. The rotation angle is (position/scale)*omega -- T is
+#     not in the formula, it only sizes the RoPE/KV allocation. The first run
+#     used T=131,072 to evaluate a 32,744-token prompt and paid 49 GB for it.
+#     T = next power of two >= prompt_len gives IDENTICAL angles at a quarter of
+#     the memory. (SCALE*ORIG==CP_T is a TRAINING requirement -- it keeps the
+#     full virtual range during training -- and is free to break at eval.)
+#  2. CONCURRENCY. Batch-1 decode under-uses the card; bench_conc.sh measures
+#     the real speedup rather than trusting the header's "~1.8x at 2/GPU".
+#
+# SCALE = T/1024 puts the last token near virtual 1,000 at every length, which
+# is the empirically best band (measured 2026-09-16: virtual ~1k scores 88-92).
+#
+# CAVEAT: these are OUR implementations of RULER's tasks, not the official
+# generators. Numbers are internally comparable across our models but NOT
+# directly comparable to published RULER scores.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+L=rulerfull_logs; mkdir -p "$L"; M="$L/master.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+CK=checkpoints_bluscriptcp/rank_0
+[ -x ./build/bluscriptCP_exec ] || die "binary missing"
+[ "$(stat -c%s "$HAY")" = 199754694 ] || die "haystack differs from prior evals"
+A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+J=${J:-4}
+MODELS=${MODELS:-"t64"}
+TASKS_ALL="niah_single niah_single_1 niah_single_3 niah_multikey niah_multikey_2 niah_multikey_3 \
+niah_multivalue niah_multiquery vt cwe fwe qa_1 qa_2"
+# prompt_len : T : SCALE   (T = min power of 2 >= len; SCALE = T/1024 -> virtual ~1k)
+CFG="4096:8192:4 8192:16384:8 16360:16384:16 32744:32768:32 65512:65536:64"
+
+say "=== FULL RULER: 13 tasks x 5 lengths, JOBS_PER_GPU=$J, models: $MODELS ==="
+for m in $MODELS; do
+  f="$CK/${m}.ckpt"; [ -s "$f" ] || die "$m checkpoint missing"
+  for c in $CFG; do
+    PL=${c%%:*}; rest=${c#*:}; T=${rest%%:*}; S=${rest##*:}
+    O="rf_${m}_${PL}"
+    if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+      say "  $m @ $PL cached"; continue; fi
+    say "  $m @ $PL  (T=$T x$S -> virtual $((PL/S)))  13 tasks"
+    t0=$(date +%s)
+    A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
+    CKPT_EXTRA="$m=${m}.ckpt" CKPTS="$m" TASKS="$TASKS_ALL" N=100 \
+    T=$T YARN=$S YARN_ORIG=4096 PROMPT_LEN=$PL GPUS=0 NP=1 JOBS_PER_GPU=$J OUT="$O" \
+      ./run_longeval.sh >> "$L/${m}_${PL}.log" 2>&1
+    say "    $(( ($(date +%s)-t0)/60 )) min"
+    if [ -s "$O/summary.csv" ]; then tail -n +2 "$O/summary.csv" | sed 's/^/      /' | tee -a "$M" >&2
+    else say "      NO SUMMARY -- see $L/${m}_${PL}.log"; fi
+  done
+done
+say "=== RULER TABLE (rows=task, cols=length) ==="
+{ printf "%-18s" "task"; for c in $CFG; do printf "%8s" "${c%%:*}"; done; echo; } | tee -a "$M"
+for t in $TASKS_ALL; do
+  printf "%-18s" "$t"
+  for c in $CFG; do
+    PL=${c%%:*}; v=$(grep -h "^${t}," rf_t64_${PL}/summary.csv 2>/dev/null | cut -d, -f2)
+    printf "%8s" "${v:-.}"
+  done; echo
+done | tee -a "$M"
+say "RULER FULL DONE"