Upload scripts/eval_collapse.sh with huggingface_hub

5123932 verified about 2 months ago

2.4 kB

#!/usr/bin/env bash
# Did retrieval degrade across the loss decline? 4 ckpts x 2 lengths, 8 GPUs.
# T=131072 YaRN x32 ORIG=4096, N=100, haystack 199,754,694 B (identical to all
# prior evals). Three things that bit me: PY must be the venv (python3 has no
# tiktoken), CKPT_EXTRA takes a BARE filename (the harness prepends rank_0/),
# and `local A=$1 B="${A}"` expands A before local assigns it -> set -u abort.
set -uo pipefail
cd "$(dirname "$0")"
export LD_LIBRARY_PATH="$(cd ../.. && pwd)/Tensor-Implementations/lib:$(cd ../.. && pwd)/Profiler/lib:${LD_LIBRARY_PATH:-}"
export PATH=/usr/local/cuda/bin:$PATH
export PY=/root/Adhi/BluTrain/envv/bin/python
L=eval_collapse_logs; mkdir -p "$L"; M="$L/master.log"
HAY=data/mix128_b6_35_640M/val_shard_00000.bin
A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
[ "$(stat -c%s "$HAY")" = 199754694 ] || { say "FATAL haystack differs"; exit 1; }
$PY -c "import tiktoken,numpy" || { say "FATAL venv lacks tiktoken/numpy"; exit 1; }

job(){
  local S="$1"; local PL="$2"; local G="$3"; local O="ec_s${S}_${PL}"
  [ -s "$O/summary.csv" ] && grep -q '^niah_single,' "$O/summary.csv" && { say "  s$S @$PL cached"; return; }
  A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="data/mix128_b6_35_640M" \
  CKPT_EXTRA="s${S}=t128c_s${S}.ckpt" CKPTS="s${S}" TASKS="niah_single" N=100 \
  T=131072 YARN=32 YARN_ORIG=4096 PROMPT_LEN="$PL" GPUS="$G" NP=1 OUT="$O" \
    ./run_longeval.sh > "$L/s${S}_${PL}.log" 2>&1
  say "  s$S @$PL -> $(grep -h '^niah_single,' "$O/summary.csv" 2>/dev/null | cut -d, -f2 | tr -d '\r')"
}
say "=== eval across the loss decline: 300/400/450/500 x {4096,32744} @x32 ==="
job 300  4096 0 &  job 400  4096 1 &  job 450  4096 2 &  job 500  4096 3 &
job 300 32744 4 &  job 400 32744 5 &  job 450 32744 6 &  job 500 32744 7 &
wait
say "=== RESULTS  niah_single, T=131072 x32 ORIG=4096, N=100 ==="
printf "  %-6s %8s %8s\n" "step" "4096" "32744" | tee -a "$M"
for s in 300 400 450 500; do
  a=$(grep -h '^niah_single,' "ec_s${s}_4096/summary.csv"  2>/dev/null | cut -d, -f2 | tr -d '\r')
  b=$(grep -h '^niah_single,' "ec_s${s}_32744/summary.csv" 2>/dev/null | cut -d, -f2 | tr -d '\r')
  printf "  %-6s %8s %8s\n" "$s" "${a:-.}" "${b:-.}" | tee -a "$M"
done
say "ref: run2 s300 100/89  s500 95/77 | t64 100/93 | old-t128 98/69"
say "EVAL DONE"

	#!/usr/bin/env bash
	# Did retrieval degrade across the loss decline? 4 ckpts x 2 lengths, 8 GPUs.
	# T=131072 YaRN x32 ORIG=4096, N=100, haystack 199,754,694 B (identical to all
	# prior evals). Three things that bit me: PY must be the venv (python3 has no
	# tiktoken), CKPT_EXTRA takes a BARE filename (the harness prepends rank_0/),
	# and `local A=$1 B="${A}"` expands A before local assigns it -> set -u abort.
	set -uo pipefail
	cd "$(dirname "$0")"
	export LD_LIBRARY_PATH="$(cd ../.. && pwd)/Tensor-Implementations/lib:$(cd ../.. && pwd)/Profiler/lib:${LD_LIBRARY_PATH:-}"
	export PATH=/usr/local/cuda/bin:$PATH
	export PY=/root/Adhi/BluTrain/envv/bin/python
	L=eval_collapse_logs; mkdir -p "$L"; M="$L/master.log"
	HAY=data/mix128_b6_35_640M/val_shard_00000.bin
	A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
	say(){ echo "[$(date -u +%H:%M:%S)] $*" \| tee -a "$M" >&2; }
	[ "$(stat -c%s "$HAY")" = 199754694 ] \|\| { say "FATAL haystack differs"; exit 1; }
	$PY -c "import tiktoken,numpy" \|\| { say "FATAL venv lacks tiktoken/numpy"; exit 1; }

	job(){
	local S="$1"; local PL="$2"; local G="$3"; local O="ec_s${S}_${PL}"
	[ -s "$O/summary.csv" ] && grep -q '^niah_single,' "$O/summary.csv" && { say " s$S @$PL cached"; return; }
	A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="data/mix128_b6_35_640M" \
	CKPT_EXTRA="s${S}=t128c_s${S}.ckpt" CKPTS="s${S}" TASKS="niah_single" N=100 \
	T=131072 YARN=32 YARN_ORIG=4096 PROMPT_LEN="$PL" GPUS="$G" NP=1 OUT="$O" \
	./run_longeval.sh > "$L/s${S}_${PL}.log" 2>&1
	say " s$S @$PL -> $(grep -h '^niah_single,' "$O/summary.csv" 2>/dev/null \| cut -d, -f2 \| tr -d '\r')"
	}
	say "=== eval across the loss decline: 300/400/450/500 x {4096,32744} @x32 ==="
	job 300 4096 0 & job 400 4096 1 & job 450 4096 2 & job 500 4096 3 &
	job 300 32744 4 & job 400 32744 5 & job 450 32744 6 & job 500 32744 7 &
	wait
	say "=== RESULTS niah_single, T=131072 x32 ORIG=4096, N=100 ==="
	printf " %-6s %8s %8s\n" "step" "4096" "32744" \| tee -a "$M"
	for s in 300 400 450 500; do
	a=$(grep -h '^niah_single,' "ec_s${s}_4096/summary.csv" 2>/dev/null \| cut -d, -f2 \| tr -d '\r')
	b=$(grep -h '^niah_single,' "ec_s${s}_32744/summary.csv" 2>/dev/null \| cut -d, -f2 \| tr -d '\r')
	printf " %-6s %8s %8s\n" "$s" "${a:-.}" "${b:-.}" \| tee -a "$M"
	done
	say "ref: run2 s300 100/89 s500 95/77 \| t64 100/93 \| old-t128 98/69"
	say "EVAL DONE"