Upload scripts/ppl_vs_pos.sh with huggingface_hub
Browse files- scripts/ppl_vs_pos.sh +38 -0
scripts/ppl_vs_pos.sh
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# Context-utilisation test: per-position teacher-forced NLL for the CLEAN model
|
| 3 |
+
# (global 1200, 83/70/50/31) and the DEGRADED one (global 1400, 80/54/31/18).
|
| 4 |
+
#
|
| 5 |
+
# A model that USES long context predicts better deeper into the window -- NLL
|
| 6 |
+
# falls with position. A model that has collapsed to LOCAL prediction is flat
|
| 7 |
+
# past a short prefix: extra context buys it nothing. Same data, same YaRN cache,
|
| 8 |
+
# same everything; only the weights differ.
|
| 9 |
+
#
|
| 10 |
+
# Attention weights themselves cannot be extracted here: the Hopper kernels are
|
| 11 |
+
# flash-style with online softmax and never materialise the attention matrix,
|
| 12 |
+
# and there are no dump hooks. This is the closest available direct measurement.
|
| 13 |
+
set -uo pipefail
|
| 14 |
+
cd "$(dirname "$0")"
|
| 15 |
+
ROOT="$(cd ../.. && pwd)"
|
| 16 |
+
export LD_LIBRARY_PATH="$ROOT/Tensor-Implementations/lib:$ROOT/Profiler/lib:${LD_LIBRARY_PATH:-}"
|
| 17 |
+
export OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
|
| 18 |
+
EXE=./build/bluscriptCP_exec
|
| 19 |
+
CK=checkpoints_bluscriptcp/rank_0
|
| 20 |
+
L=mixcont_logs
|
| 21 |
+
T=65536; NP=8; CPSZ=8
|
| 22 |
+
ARCH="CP_N_EMBD=1536 CP_N_LAYER=36 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=0"
|
| 23 |
+
BASE="CP_SIZE=$CPSZ CP_ATTN_MODE=ring CP_ATTN_FUSION=split CP_T=$T \
|
| 24 |
+
YARN_SCALE=2 YARN_ORIG_MAXPOS=32768 CP_BF16=1 CP_ZERO=1 CP_ZERO_V2=1 CP_ZERO_SYMMETRIC=1"
|
| 25 |
+
run_one(){ # $1=label $2=ckpt
|
| 26 |
+
echo "[$(date -u +%H:%M:%S)] PPL-vs-position: $1 ($2)" >&2
|
| 27 |
+
env $ARCH $BASE CP_EVAL_PPL=1 CP_EVAL_WINDOWS=${W:-8} \
|
| 28 |
+
CP_EVAL_OUT="$L/ppl_$1.csv" \
|
| 29 |
+
CP_INIT_FROM="$PWD/$CK/$2" CP_DATA_ROOT="$PWD/Data_Loader/Data/mixcont" \
|
| 30 |
+
CP_B=1 CP_GLOBAL_BATCH=$T CP_MAX_STEPS=1 \
|
| 31 |
+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 mpirun --allow-run-as-root -np $NP "$EXE" \
|
| 32 |
+
> "$L/ppl_$1.log" 2>&1
|
| 33 |
+
if [ -s "$L/ppl_$1.csv" ]; then echo " -> $L/ppl_$1.csv ($(wc -l < "$L/ppl_$1.csv") rows)" >&2
|
| 34 |
+
else echo " -> FAILED; tail:" >&2; tail -12 "$L/ppl_$1.log" >&2; fi
|
| 35 |
+
}
|
| 36 |
+
run_one clean1200 mix64k_step1200.ckpt
|
| 37 |
+
run_one degraded1400 cliff_g1400.ckpt
|
| 38 |
+
echo "[$(date -u +%H:%M:%S)] PPL EVAL DONE" >&2
|