Upload scripts/run_ruler.sh with huggingface_hub

scripts/run_ruler.sh

@@ -22,8 +22,10 @@ cd "$(dirname "$0")"
 export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
 source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
 
-RULER="niah_single niah_single_1 niah_single_3 niah_multikey niah_multikey_2 \
-niah_multikey_3 niah_multivalue niah_multiquery vt cwe fwe qa_1 qa_2"
+# Full 13. Override with RULER_TASKS="niah_single" for a NIAH-only pass
+# (3 jobs/length instead of 39 -> 1 wave on 8 GPUs instead of 5).
+RULER=${RULER_TASKS:-"niah_single niah_single_1 niah_single_3 niah_multikey niah_multikey_2 \
+niah_multikey_3 niah_multivalue niah_multiquery vt cwe fwe qa_1 qa_2"}
 LENS=${LENS:-"4096 8192 16384 32744 65512"}
 N=${N:-100}
 GPUS=${GPUS:-0,1,2,3,4,5,6,7}
@@ -41,17 +43,19 @@ grep -q 'CP_FUSED_ROPE=1' build/objects/.cp_rope_flags && \
 grep -q 'CP_ATTN_SPLIT=1' build/objects/.cp_rope_flags || die "binary lacks split kernels"
 [ -f "$HAY" ] || die "haystack missing: $HAY"
 [ "$(stat -c%s "$HAY")" = 199754694 ] || die "haystack is NOT the file every prior eval used"
-for f in Tests/bluscriptcp/qa_data/squad_val.parquet Tests/bluscriptcp/qa_data/hotpot_val.parquet; do
-  [ -f "$f" ] || die "QA data missing: $f  (qa_1/qa_2 cannot run)"; done
-python -c "import pyarrow" 2>/dev/null || die "pyarrow missing (needed by qa_1/qa_2)"
+if echo "$RULER" | grep -q 'qa_'; then
+  for f in Tests/bluscriptcp/qa_data/squad_val.parquet Tests/bluscriptcp/qa_data/hotpot_val.parquet; do
+    [ -f "$f" ] || die "QA data missing: $f  (qa_1/qa_2 cannot run)"; done
+  python -c "import pyarrow" 2>/dev/null || die "pyarrow missing (needed by qa_1/qa_2)"
+fi
 for m in $MODELS; do
   f=$(echo "$EXTRA" | tr ' ' '\n' | grep "^$m=" | cut -d= -f2)
   [ -f "$CK/$f" ] || die "checkpoint for '$m' missing: $CK/$f"
 done
 
-say "=== FULL RULER: 13 tasks x $(echo $MODELS|wc -w) models x $(echo $LENS|wc -w) lengths, N=$N ==="
+say "=== RULER: $(echo $RULER|wc -w) task(s) x $(echo $MODELS|wc -w) models x $(echo $LENS|wc -w) lengths, N=$N ==="
 say "models: $MODELS"
-say "jobs per length: $(( 13 * $(echo $MODELS|wc -w) ))  across $(echo $GPUS|tr ',' ' '|wc -w) GPUs"
+say "jobs per length: $(( $(echo $RULER|wc -w) * $(echo $MODELS|wc -w) ))  across $(echo $GPUS|tr ',' ' '|wc -w) GPUs"
 
 for PL in $LENS; do
   O="ruler_len${PL}"
@@ -66,7 +70,7 @@ for PL in $LENS; do
   T=262144 YARN=64 YARN_ORIG=4096 PROMPT_LEN=$PL GPUS="$GPUS" NP=1 OUT="$O" \
     ./run_longeval.sh >> "$L/len_${PL}.log" 2>&1
   if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
-    say "  done: $(( $(wc -l < "$O/summary.csv") - 1 ))/13 tasks scored"
+    say "  done: $(( $(wc -l < "$O/summary.csv") - 1 ))/$(echo $RULER|wc -w) tasks scored"
     tail -n +2 "$O/summary.csv" | sed 's/^/    /' | tee -a "$M" >&2
   else
     say "  FAILED at len $PL"; grep -ahA2 'what()' "$O"/gen/*.log 2>/dev/null | head -4 | tee -a "$M" >&2