Upload scripts/eval_long.sh with huggingface_hub

25f79e5 verified about 2 months ago

2.11 kB

#!/usr/bin/env bash
# THE MISSING EVAL: 65,512 and 131,000 -- the lengths this rung actually targets.
# Everything measured so far was 4,096 / 32,744, i.e. a quarter of the window.
# T=131072 YaRN x32 ORIG=4096 -> virtual 2,047 @65,512 and 4,094 @131,000.
# 131,000 sits at the very edge of the 4,096 pretrain range, so expect a drop
# for every model; the comparison between models is what matters.
set -uo pipefail; cd "$(dirname "$0")"
export LD_LIBRARY_PATH="$(cd ../.. && pwd)/Tensor-Implementations/lib:$(cd ../.. && pwd)/Profiler/lib:${LD_LIBRARY_PATH:-}"
export PATH=/usr/local/cuda/bin:$PATH; export PY=/root/Adhi/BluTrain/envv/bin/python
L=eval_long_logs; mkdir -p "$L"; M="$L/master.log"
A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
EX="c300=t128c_s300.ckpt d300=t128d_s300.ckpt d360=t128d_s360.ckpt"
NM="c300 d300 d360"
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
blk(){ local PL="$1"; local JPG="$2"; local O="el_${PL}"
  [ -s "$O/summary.csv" ] && grep -q '^niah_single,' "$O/summary.csv" && { say "  $PL cached"; return; }
  A1B="$A680" HAY="data/mix128_b6_60/val_shard_00000.bin" GEN_DATA_ROOT="data/mix128_b6_60" \
  CKPT_EXTRA="$EX" CKPTS="$NM" TASKS="niah_single" N=100 \
  T=131072 YARN=32 YARN_ORIG=4096 PROMPT_LEN="$PL" GPUS="0,1,2,3" NP=1 JOBS_PER_GPU="$JPG" OUT="$O" \
    ./run_longeval.sh > "$L/len_${PL}.log" 2>&1
  say "  len $PL: $(head -1 $O/summary.csv 2>/dev/null|tr -d '\r') -> $(grep -h '^niah_single,' $O/summary.csv 2>/dev/null|tr -d '\r')"; }
say "=== LONG eval: c300 / d300 / d360 at 65,512 then 131,000 ==="
blk 65512 2                 # 3 jobs, 8 slots
say "  65k done, starting 131k at JOBS_PER_GPU=1 (4x the activations)"
blk 131000 1                # 3 jobs, 4 slots -- conservative on memory
say "=== LONG RESULTS ==="
for PL in 65512 131000; do
  say "  $PL: $(head -1 el_${PL}/summary.csv 2>/dev/null|tr -d '\r') -> $(grep -h '^niah_single,' el_${PL}/summary.csv 2>/dev/null|tr -d '\r')"
done
say "ref @65,512 x32: t64=63  old-t128=31   (@131,000 never measured for anything)"
say "EVAL_LONG_DONE"

	#!/usr/bin/env bash
	# THE MISSING EVAL: 65,512 and 131,000 -- the lengths this rung actually targets.
	# Everything measured so far was 4,096 / 32,744, i.e. a quarter of the window.
	# T=131072 YaRN x32 ORIG=4096 -> virtual 2,047 @65,512 and 4,094 @131,000.
	# 131,000 sits at the very edge of the 4,096 pretrain range, so expect a drop
	# for every model; the comparison between models is what matters.
	set -uo pipefail; cd "$(dirname "$0")"
	export LD_LIBRARY_PATH="$(cd ../.. && pwd)/Tensor-Implementations/lib:$(cd ../.. && pwd)/Profiler/lib:${LD_LIBRARY_PATH:-}"
	export PATH=/usr/local/cuda/bin:$PATH; export PY=/root/Adhi/BluTrain/envv/bin/python
	L=eval_long_logs; mkdir -p "$L"; M="$L/master.log"
	A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
	EX="c300=t128c_s300.ckpt d300=t128d_s300.ckpt d360=t128d_s360.ckpt"
	NM="c300 d300 d360"
	say(){ echo "[$(date -u +%H:%M:%S)] $*" \| tee -a "$M" >&2; }
	blk(){ local PL="$1"; local JPG="$2"; local O="el_${PL}"
	[ -s "$O/summary.csv" ] && grep -q '^niah_single,' "$O/summary.csv" && { say " $PL cached"; return; }
	A1B="$A680" HAY="data/mix128_b6_60/val_shard_00000.bin" GEN_DATA_ROOT="data/mix128_b6_60" \
	CKPT_EXTRA="$EX" CKPTS="$NM" TASKS="niah_single" N=100 \
	T=131072 YARN=32 YARN_ORIG=4096 PROMPT_LEN="$PL" GPUS="0,1,2,3" NP=1 JOBS_PER_GPU="$JPG" OUT="$O" \
	./run_longeval.sh > "$L/len_${PL}.log" 2>&1
	say " len $PL: $(head -1 $O/summary.csv 2>/dev/null\|tr -d '\r') -> $(grep -h '^niah_single,' $O/summary.csv 2>/dev/null\|tr -d '\r')"; }
	say "=== LONG eval: c300 / d300 / d360 at 65,512 then 131,000 ==="
	blk 65512 2 # 3 jobs, 8 slots
	say " 65k done, starting 131k at JOBS_PER_GPU=1 (4x the activations)"
	blk 131000 1 # 3 jobs, 4 slots -- conservative on memory
	say "=== LONG RESULTS ==="
	for PL in 65512 131000; do
	say " $PL: $(head -1 el_${PL}/summary.csv 2>/dev/null\|tr -d '\r') -> $(grep -h '^niah_single,' el_${PL}/summary.csv 2>/dev/null\|tr -d '\r')"
	done
	say "ref @65,512 x32: t64=63 old-t128=31 (@131,000 never measured for anything)"
	say "EVAL_LONG_DONE"