File size: 2,271 Bytes
a99d9df | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | #!/usr/bin/env bash
# Eval the checkpoint at the cliff ONSET (local step 200 = global 1400) against
# its own parent (mix64k step 1200 = 83/70/50/31, measured on this harness).
# The question: has the loss cliff actually cost capability yet, or only loss?
set -uo pipefail
cd "$(dirname "$0")"
source /root/Adhi/BluTrain/envv/bin/activate
export LD_LIBRARY_PATH="/root/Adhi/BluTrain/Tensor-Implementations/lib:/root/Adhi/BluTrain/Profiler/lib:${LD_LIBRARY_PATH:-}"
PY=python
T=65536; ORIG=32768; SCALE=2
DATA=Data_Loader/Data/mixcont
HAY="$PWD/Data_Loader/Data/b2_8192_16384/val_shard_00000.bin"
L=mixcont_logs; M="$L/eval.log"; : > "$M"
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
say "=== eval global step 1400 (cliff onset) -- 4 lengths, one GPU each ==="
for LEN in 8192 16360 32744 65512; do
( GEN_DATA_ROOT=$DATA HAY="$HAY" PY=$PY OUT=cliff1400_len_$LEN TASKS="niah_single" \
CKPTS="c1400" N=100 PROMPT_LEN=$LEN MAX_NEW=24 T=$T \
DEPTHS="0.0,0.25,0.5,0.75,1.0" YARN_ORIG=$ORIG YARN=$SCALE \
GPUS=$(( (LEN==8192)?0:(LEN==16360)?1:(LEN==32744)?2:3 )) JOBS_PER_GPU=1 \
BF16=1 GREEDY=1 CKPT_EXTRA="c1400=cliff_g1400.ckpt" ./run_longeval.sh \
>> "$L/eval_$LEN.log" 2>&1 ) &
done
wait
say "=== results vs parent (mix64k step 1200) ==="
$PY - <<'PY' | tee -a "$M"
import csv, math, os
P={8192:83.0,16360:70.0,32744:50.0,65512:31.0} # the parent, measured
def w(p,n=100):
z=1.96; ph=p/100
return 100*z*math.sqrt(ph*(1-ph)/n+z*z/(4*n*n))/(1+z*z/n)
print(f" {'ctx':>7} {'parent':>7} {'g1400':>7} {'delta':>7} {'bar':>6} verdict")
for L in (8192,16360,32744,65512):
f=f"cliff1400_len_{L}/summary.csv"; v=None
if os.path.exists(f):
for r in csv.reader(open(f)):
if r and r[0].strip()=="niah_single": v=float(r[1])
if v is None: print(f" {L:>7} {P[L]:>7.1f} {'-':>7} {'-':>7} {'-':>6} not scored"); continue
b=math.hypot(w(v),w(P[L])); d=v-P[L]
print(f" {L:>7} {P[L]:>7.1f} {v:>7.1f} {d:>+7.1f} {b:>6.1f} {'DEGRADED' if -d>b else 'ok'}")
PY
say "=== depth profiles ==="
for LEN in 8192 16360 32744 65512; do
f=cliff1400_len_$LEN/depth_summary.csv
[ -s "$f" ] && { printf " %6s: " "$LEN" >&2; awk -F, 'NR>1{printf "%5.1f ",$5}' "$f" >&2; echo >&2; }
done
say "=== EVAL DONE ==="
|