File size: 2,431 Bytes
f827ec2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | #!/usr/bin/env bash
# T=131,072 / YaRN 32 x 4096. A PRE-REGISTERED test of the virtual-position model.
#
# Hypothesis: score is governed by where the last token lands inside the 4,096
# range the model was pretrained on (real position / scale), not by token count.
#
# PREDICTIONS, written before the run:
# 4,096 tok -> virtual 128 : 92-99 (was 92 at x16)
# 32,744 tok -> virtual 1,023 : 85-92 (was 54 at x16) <- the big one
# 65,512 tok -> virtual 2,047 : 45-55 (was 32 at x16)
# If 32,744 does NOT jump from 54 into the 80s, the virtual-position model is
# wrong and the gains so far were something else.
set -uo pipefail
cd "$(dirname "$0")"
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
L=e128k_logs; mkdir -p "$L"; M="$L/master.log"
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
T=131072; SCALE=32; ORIG=4096
[ $((SCALE*ORIG)) -eq $T ] || { say "FATAL YaRN invariant"; exit 1; }
HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
[ "$(stat -c%s "$HAY")" = 199754694 ] || { say "FATAL haystack differs"; exit 1; }
A1B_680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
say "=== T=$T YaRN ${SCALE}x${ORIG} -- predictions: 4096->92-99, 32744->85-92, 65512->45-55 ==="
for PL in 4096 32744 65512; do
O="e128k_len${PL}"
[ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && {
say " len $PL cached: $(grep '^niah_single,' $O/summary.csv)"; continue; }
say "eval prompt_len=$PL (virtual $((PL/SCALE)))"
A1B="$A1B_680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
CKPT_EXTRA="p1final=blumodelcp_run12_step_1200.ckpt base680=varybatch.ckpt" \
CKPTS="p1final base680" TASKS=niah_single N=100 \
T=$T YARN=$SCALE YARN_ORIG=$ORIG PROMPT_LEN=$PL GPUS=0,1 NP=1 OUT="$O" \
./run_longeval.sh >> "$L/eval_${PL}.log" 2>&1
if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
say " len $PL (virtual $((PL/SCALE))) -> $(head -1 $O/summary.csv) | $(grep '^niah_single,' $O/summary.csv)"
else
say " len $PL FAILED"; grep -ahA2 'what()\|out of memory' "$O"/gen/*.log 2>/dev/null|head -4|tee -a "$M" >&2
fi
done
say "=== RESULTS T=$T (cols base680,p1final) ==="
for PL in 4096 32744 65512; do
f="e128k_len${PL}/summary.csv"
[ -f "$f" ] && say " len $PL virtual $((PL/SCALE)) $(grep '^niah_single,' $f)"
done
say "DONE"
|