Upload scripts/peer_zeros.sh with huggingface_hub
Browse files- scripts/peer_zeros.sh +56 -0
scripts/peer_zeros.sh
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# =============================================================================
|
| 3 |
+
# THE ZEROS CHECK. t64 scores EXACTLY 0.0 on niah_multivalue, niah_multiquery,
|
| 4 |
+
# cwe and fwe at all three measured lengths. A uniform 0.0 is as often a
|
| 5 |
+
# prompt-format or scoring bug as a real capability floor -- and cwe/fwe run
|
| 6 |
+
# through a BODY_TASKS code path that has never been exercised before today.
|
| 7 |
+
#
|
| 8 |
+
# Qwen3-0.6B-Base is the control: a size-matched peer that flat-maxed
|
| 9 |
+
# niah_single (100% to 32,736) in the 2026-09-09 sweep.
|
| 10 |
+
# Qwen scores well -> our 0.0 is a real capability floor.
|
| 11 |
+
# Qwen also scores 0 -> the TASK is broken, and our ~25 RULER average is wrong.
|
| 12 |
+
#
|
| 13 |
+
# Starts with multivalue + vt because peer_gen.py already implements them; cwe,
|
| 14 |
+
# fwe and multiquery would need porting and are only worth it if this is clean.
|
| 15 |
+
# niah_single is the control that proves the pipeline works at all.
|
| 16 |
+
# =============================================================================
|
| 17 |
+
set -uo pipefail
|
| 18 |
+
cd "$(dirname "$0")/Tests/bluscriptcp/peer"
|
| 19 |
+
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
|
| 20 |
+
L=../../../peer_zeros_logs; mkdir -p "$L"; M="$L/master.log"
|
| 21 |
+
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
|
| 22 |
+
|
| 23 |
+
say "waiting for the t64 RULER suite to release the GPU..."
|
| 24 |
+
# Wait on the SUITE PROCESS, not the GPU. The suite has brief gaps between
|
| 25 |
+
# length blocks where no exec is running and GPU memory drops -- a GPU-free
|
| 26 |
+
# check slips through exactly there (it did, at 14:42:31). The pid is the
|
| 27 |
+
# only signal that covers the whole run.
|
| 28 |
+
RPID=$(cat ../../../rulerfull.pid 2>/dev/null || cat rulerfull.pid 2>/dev/null || echo "")
|
| 29 |
+
if [ -n "$RPID" ]; then
|
| 30 |
+
say "waiting on RULER suite pid $RPID"
|
| 31 |
+
while kill -0 "$RPID" 2>/dev/null; do sleep 60; done
|
| 32 |
+
fi
|
| 33 |
+
while ps -eo args --no-headers | grep -q "[b]luscriptCP_exec"; do sleep 30; done
|
| 34 |
+
say "GPU free"
|
| 35 |
+
|
| 36 |
+
export PY=/root/Adhi/BluTrain/envv/bin/python
|
| 37 |
+
ROOT=$PWD/peer_out; mkdir -p "$ROOT"
|
| 38 |
+
HAY=$ROOT/haystack.txt
|
| 39 |
+
if [ ! -s "$HAY" ]; then
|
| 40 |
+
say "building shared text haystack (4M tokens)"
|
| 41 |
+
$PY make_haystack.py --out "$HAY" --tokens 4000000 >>"$M" 2>&1 || { say "haystack FAILED"; exit 1; }
|
| 42 |
+
fi
|
| 43 |
+
say "haystack $(stat -Lc%s "$HAY") bytes"
|
| 44 |
+
|
| 45 |
+
MODELS=${MODELS:-"qwen3-0.6b"} \
|
| 46 |
+
TASKS=${TASKS:-"niah_single niah_multivalue vt"} \
|
| 47 |
+
LENS=${LENS:-"4096 16384"} \
|
| 48 |
+
N=${N:-100} GPUS=${GPUS:-0} ROOT="$ROOT" HAY="$HAY" \
|
| 49 |
+
./run_peer_eval.sh >> "$L/peer.log" 2>&1
|
| 50 |
+
say "=== PEER RESULTS (Qwen3-0.6B-Base, our task implementations) ==="
|
| 51 |
+
find "$ROOT" -name '*score*.csv' -o -name 'summary*.csv' 2>/dev/null | while read f; do
|
| 52 |
+
say " $(echo $f|sed "s|$ROOT/||"): $(tail -n +2 "$f" 2>/dev/null | tr '\n' ' ')"
|
| 53 |
+
done
|
| 54 |
+
say "=== OUR t64 for comparison ==="
|
| 55 |
+
say " niah_single 87/91/92 | niah_multivalue 0/0/0 | vt 3/3/2 (at 4096/8192/16360)"
|
| 56 |
+
say "PEER DONE"
|