Upload scripts/t64_finish.sh with huggingface_hub

scripts/t64_finish.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Wait for the 64k run -> three-way eval -> upload verified -> teardown verdict.
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+REPO=unparallelled/BluScriptCP
+L=t64_logs; M="$L/finish.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+
+say "waiting for training to finish writing..."
+while ! grep -qa '64K TRAINING COMPLETE\|FATAL' "$L/master.log" 2>/dev/null; do
+  pgrep -f bluscriptCP_exec >/dev/null 2>&1 || { sleep 40
+    grep -qa '64K TRAINING COMPLETE\|FATAL' "$L/master.log" 2>/dev/null || break; }
+  sleep 30
+done
+grep -qa FATAL "$L/master.log" && say "training FAILED"
+
+CK=checkpoints_bluscriptcp/rank_0
+RN=$(cat "$L/runnum" 2>/dev/null || echo "")
+FINAL=""
+[ -n "$RN" ] && FINAL=$(ls -t $CK/blumodelcp_run${RN}_step_*.ckpt 2>/dev/null | head -1)
+[ -z "$FINAL" ] && FINAL=$(ls -t $CK/blumodelcp_run*_step_*.ckpt 2>/dev/null | head -1)
+[ -z "$FINAL" ] && { say "NO CHECKPOINT -- cannot evaluate"; exit 1; }
+say "final: $(basename "$FINAL")  $(stat -c%s "$FINAL") B"
+
+A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+[ "$(stat -c%s "$HAY")" = 199754694 ] || { say "FATAL haystack differs from every prior eval"; exit 1; }
+EX="t64=$(basename "$FINAL") p1final=p1final.ckpt base680=varybatch.ckpt"
+ev(){ # T SCALE PL TASKS tag
+  local T=$1 S=$2 PL=$3 TK=$4 TG=$5
+  local O="t64ev_${TG}_${PL}"
+  [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && { say "  $TG $PL cached"; return 0; }
+  say "  eval $TG len $PL (T=$T x$S -> virtual $((PL/S))) [$TK]"
+  A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
+  CKPT_EXTRA="$EX" CKPTS="t64 p1final base680" TASKS="$TK" N=100 \
+  T=$T YARN=$S YARN_ORIG=4096 PROMPT_LEN=$PL GPUS=0,1,2,3,4,5,6,7 NP=1 OUT="$O" \
+    ./run_longeval.sh >> "$L/ev_${TG}_${PL}.log" 2>&1
+  if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+    say "    $(head -1 $O/summary.csv)"; tail -n +2 "$O/summary.csv"|sed 's/^/      /'|tee -a "$M" >&2
+  else say "    FAILED"; grep -ahA2 'what()' "$O"/gen/*.log 2>/dev/null|head -3|tee -a "$M" >&2; fi
+}
+say "=== THREE-WAY EVAL: t64 vs p1final vs base680 ==="
+ev 262144 64  4096 niah_single x64
+ev 262144 64 32744 niah_single x64
+ev 262144 64 65512 niah_single x64
+ev  65536 16 65512 niah_single x16at
+say "=== harder tasks -- if NIAH rises and these do not, it is a benchmark gain ==="
+ev 262144 64 32744 "niah_multikey val_backward" hard
+
+up(){ [ -f "$1" ] || { say "  MISSING $1"; return 1; }
+  hf upload "$REPO" "$1" "$2" >>"$L/upload.log" 2>&1
+  python - "$REPO" "$1" "$2" <<'V'
+import sys,os
+from huggingface_hub import HfApi
+w=os.path.getsize(sys.argv[2])
+g={f.rfilename:(f.size or 0) for f in HfApi().repo_info(sys.argv[1],files_metadata=True).siblings}.get(sys.argv[3])
+ok=(g==w); print(("  VERIFIED  " if ok else "  *** FAILED ")+f"{sys.argv[3]} local={w:,} remote={g}")
+sys.exit(0 if ok else 1)
+V
+}
+BAD=0
+say "=== UPLOAD ==="
+for f in t64_train.sh t64_finish.sh dl_64k.sh; do up "$f" "scripts/$f" || BAD=$((BAD+1)); done
+up Data_Loader/Data/mix64/MANIFEST.txt data/mix64_MANIFEST.txt || BAD=$((BAD+1))
+for g in master.log train.log; do cp -f "$L/$g" "$L/$g.snap"; up "$L/$g.snap" "logs/t64_$g" || BAD=$((BAD+1)); done
+for d in t64ev_*; do [ -f "$d/summary.csv" ] || continue
+  up "$d/summary.csv" "evals/2026-09-17/${d}_summary.csv" || BAD=$((BAD+1)); done
+ST=$(basename "$FINAL" | grep -oE 'step_[0-9]+' | cut -d_ -f2)
+up "$FINAL" "checkpoints/t64_680m_step${ST}_rank0.ckpt" || BAD=$((BAD+1))
+say "=== TEARDOWN VERDICT ==="
+[ "$BAD" -eq 0 ] && say "ALL VERIFIED -- SAFE TO TEAR DOWN" || say "*** $BAD failure(s) -- DO NOT TEAR DOWN ***"