Upload scripts/vb_diag.sh with huggingface_hub

scripts/vb_diag.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# =============================================================================
+# THE DIAGNOSTIC: is val_backward ~0 because the model cannot do backward
+# lookup AT ALL, or only because it cannot do it at LONG context?
+#
+#   fails at 1k AND 32k  -> missing circuit (anti-induction). Reverse-training
+#                           data is the right lever, and it can be trained SHORT
+#                           and cheap.
+#   fine at 1k, dies long -> retrieval-RANGE problem. Reverse training will not
+#                           touch it and the 4-hour experiment is not worth running.
+#
+# niah_single runs alongside as the forward control at every length: an induction
+# head does match-and-copy ([.. A B .. A] -> B), which is exactly niah_single.
+# val_backward asks for the mirror (find B, emit the preceding A). Measuring both
+# at the same length isolates the forward/backward asymmetry from raw difficulty.
+#
+# Scale is held at the model's best-known config per length so we are not
+# re-measuring the config effect: x4 at 1k/4k, x8 at 8k, x16 at 16k.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+L=vb_logs; mkdir -p "$L"; M="$L/master.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+die(){ say "FATAL: $*"; exit 1; }
+EXE=./build/bluscriptCP_exec
+HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+CK=checkpoints_bluscriptcp/rank_0
+[ -x "$EXE" ] || die "$EXE not built"
+grep -q 'CP_FUSED_ROPE=1' build/objects/.cp_rope_flags || die "binary lacks split kernels"
+[ "$(stat -c%s "$HAY")" = 199754694 ] || die "haystack differs from prior evals"
+[ -s "$CK/p1final.ckpt" ] && [ -s "$CK/varybatch.ckpt" ] || die "checkpoints missing"
+A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+EX="p1final=p1final.ckpt base680=varybatch.ckpt"
+
+run(){ # T SCALE PL
+  local T=$1 S=$2 PL=$3
+  local O="vb_len${PL}"
+  [ $((S*4096)) -eq $T ] || { say "YaRN invariant $S*4096 != $T"; return 1; }
+  [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && {
+    say "  len $PL cached"; return 0; }
+  say "  len $PL  (T=$T x$S -> virtual $((PL/S)))  [val_backward + niah_single]"
+  A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
+  CKPT_EXTRA="$EX" CKPTS="p1final base680" TASKS="val_backward niah_single" N=100 \
+  T=$T YARN=$S YARN_ORIG=4096 PROMPT_LEN=$PL GPUS=0 NP=1 OUT="$O" \
+    ./run_longeval.sh >> "$L/ev_${PL}.log" 2>&1
+  if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+    say "    $(head -1 $O/summary.csv)"; tail -n +2 "$O/summary.csv"|sed 's/^/      /'|tee -a "$M" >&2
+  else say "    FAILED"; grep -ahA2 'what()' "$O"/gen/*.log 2>/dev/null|head -4|tee -a "$M" >&2; fi
+}
+say "=== val_backward diagnostic: circuit deficit or range problem? ==="
+run 16384  4  1024
+run 16384  4  4096
+run 32768  8  8192
+run 65536 16 16360
+say "=== SUMMARY (cols: base680, p1final) ==="
+for PL in 1024 4096 8192 16360; do
+  f="vb_len${PL}/summary.csv"
+  [ -f "$f" ] && say "  len $PL  $(head -1 $f) | $(tail -n +2 $f | tr '\n' ' ')"
+done
+say "DONE"