Upload scripts/e64k.sh with huggingface_hub

ca421d1 verified about 2 months ago

2.41 kB

#!/usr/bin/env bash
# T=65,536 / YaRN 16 x 4096. Does the scale trend continue, or break?
#
# MEMORY: this is INFERENCE, not training. The 64k TRAINING figure (~117 GB at
# cp=2, hence >=4 GPUs) covers activations for backward, gradients and optimizer
# state -- none of which exist here. Measured at T=32,768: 3,823 MiB/GPU. KV
# cache at 65,536 is 3.22 GB + 1.36 GB weights, so cp=1 per job on an 80 GB card.
#
# 4,096 is included deliberately: at scale 16 the interpolation is most
# aggressive, so if over-compression costs short-context resolution it shows
# there first.
set -uo pipefail
cd "$(dirname "$0")"
export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
L=e64k_logs; mkdir -p "$L"; M="$L/master.log"
say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
T=65536; SCALE=16; ORIG=4096
[ $((SCALE*ORIG)) -eq $T ] || { say "FATAL YaRN invariant"; exit 1; }
HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
[ "$(stat -c%s "$HAY")" = 199754694 ] || { say "FATAL haystack differs from phase 1's"; exit 1; }
A1B_680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
say "=== T=$T YaRN ${SCALE}x${ORIG} -- base680 (control) vs p1final ==="
for PL in 4096 16360 32744 65512; do
  O="e64k_len${PL}"
  [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && {
    say "  len $PL cached: $(grep '^niah_single,' $O/summary.csv)"; continue; }
  say "eval prompt_len=$PL"
  A1B="$A1B_680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
  CKPT_EXTRA="p1final=blumodelcp_run12_step_1200.ckpt base680=varybatch.ckpt" \
  CKPTS="p1final base680" TASKS=niah_single N=100 \
  T=$T YARN=$SCALE YARN_ORIG=$ORIG PROMPT_LEN=$PL GPUS=0,1 NP=1 OUT="$O" \
    ./run_longeval.sh >> "$L/eval_${PL}.log" 2>&1
  if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
    say "  len $PL -> $(head -1 $O/summary.csv) | $(grep '^niah_single,' $O/summary.csv)"
    grep -ahoiE 'out of memory|bad_alloc' "$O"/gen/*.log 2>/dev/null | head -1 | tee -a "$M" >&2
  else
    say "  len $PL FAILED"; grep -ahA2 'what()' "$O"/gen/*.log 2>/dev/null | head -4 | tee -a "$M" >&2
  fi
done
say "=== RESULTS T=$T (cols base680,p1final) ==="
for PL in 4096 16360 32744 65512; do
  f="e64k_len${PL}/summary.csv"
  [ -f "$f" ] && say "  len $PL  $(head -1 $f) -> $(grep '^niah_single,' $f)"
done
say "DONE"

	#!/usr/bin/env bash
	# T=65,536 / YaRN 16 x 4096. Does the scale trend continue, or break?
	#
	# MEMORY: this is INFERENCE, not training. The 64k TRAINING figure (~117 GB at
	# cp=2, hence >=4 GPUs) covers activations for backward, gradients and optimizer
	# state -- none of which exist here. Measured at T=32,768: 3,823 MiB/GPU. KV
	# cache at 65,536 is 3.22 GB + 1.36 GB weights, so cp=1 per job on an 80 GB card.
	#
	# 4,096 is included deliberately: at scale 16 the interpolation is most
	# aggressive, so if over-compression costs short-context resolution it shows
	# there first.
	set -uo pipefail
	cd "$(dirname "$0")"
	export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
	source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
	L=e64k_logs; mkdir -p "$L"; M="$L/master.log"
	say(){ echo "[$(date -u +%H:%M:%S)] $*" \| tee -a "$M" >&2; }
	T=65536; SCALE=16; ORIG=4096
	[ $((SCALE*ORIG)) -eq $T ] \|\| { say "FATAL YaRN invariant"; exit 1; }
	HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
	[ "$(stat -c%s "$HAY")" = 199754694 ] \|\| { say "FATAL haystack differs from phase 1's"; exit 1; }
	A1B_680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
	say "=== T=$T YaRN ${SCALE}x${ORIG} -- base680 (control) vs p1final ==="
	for PL in 4096 16360 32744 65512; do
	O="e64k_len${PL}"
	[ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && {
	say " len $PL cached: $(grep '^niah_single,' $O/summary.csv)"; continue; }
	say "eval prompt_len=$PL"
	A1B="$A1B_680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
	CKPT_EXTRA="p1final=blumodelcp_run12_step_1200.ckpt base680=varybatch.ckpt" \
	CKPTS="p1final base680" TASKS=niah_single N=100 \
	T=$T YARN=$SCALE YARN_ORIG=$ORIG PROMPT_LEN=$PL GPUS=0,1 NP=1 OUT="$O" \
	./run_longeval.sh >> "$L/eval_${PL}.log" 2>&1
	if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
	say " len $PL -> $(head -1 $O/summary.csv) \| $(grep '^niah_single,' $O/summary.csv)"
	grep -ahoiE 'out of memory\|bad_alloc' "$O"/gen/*.log 2>/dev/null \| head -1 \| tee -a "$M" >&2
	else
	say " len $PL FAILED"; grep -ahA2 'what()' "$O"/gen/*.log 2>/dev/null \| head -4 \| tee -a "$M" >&2
	fi
	done
	say "=== RESULTS T=$T (cols base680,p1final) ==="
	for PL in 4096 16360 32744 65512; do
	f="e64k_len${PL}/summary.csv"
	[ -f "$f" ] && say " len $PL $(head -1 $f) -> $(grep '^niah_single,' $f)"
	done
	say "DONE"