Upload scripts/t32_finish.sh with huggingface_hub

scripts/t32_finish.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Wait for 32k training -> evaluate -> upload everything verified -> teardown verdict.
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+REPO=unparallelled/BluScriptCP
+L=t32_logs; M="$L/finish.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+
+say "waiting for training..."
+while ! grep -qa '32K TRAINING COMPLETE\|FATAL' "$L/master.log" 2>/dev/null; do
+  pgrep -f bluscriptCP_exec >/dev/null 2>&1 || { sleep 30
+    grep -qa '32K TRAINING COMPLETE\|FATAL' "$L/master.log" 2>/dev/null || {
+      say "trainer vanished without COMPLETE"; break; }; }
+  sleep 60
+done
+if grep -qa FATAL "$L/master.log" 2>/dev/null; then say "training FAILED"; fi
+
+CK=checkpoints_bluscriptcp/rank_0
+RN=$(cat "$L/runnum" 2>/dev/null || echo "")
+FINAL=""
+[ -n "$RN" ] && FINAL=$(ls -t $CK/blumodelcp_run${RN}_step_*.ckpt 2>/dev/null | head -1)
+[ -z "$FINAL" ] && FINAL=$(ls -t $CK/blumodelcp_run*_step_*.ckpt 2>/dev/null | grep -v run12 | head -1)
+if [ -z "$FINAL" ]; then say "NO CHECKPOINT FOUND -- nothing to evaluate"; else
+  say "final: $(basename "$FINAL") ($(stat -c%s "$FINAL") B)"
+  A680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+  HAY=Data_Loader/Data/b2_8192_16384/val_shard_00000.bin
+  EX="t32=$(basename "$FINAL") p1final=blumodelcp_run12_step_1200.ckpt base680=varybatch.ckpt"
+  ev(){ # T SCALE PL TASKS tag
+    local T=$1 S=$2 PL=$3 TK=$4
+    local TG=$5 O="t32ev_${5}_${3}"
+    [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ] && return 0
+    say "  eval $TG len $PL (T=$T x$S -> virtual $((PL/S))) [$TK]"
+    A1B="$A680" HAY="$HAY" GEN_DATA_ROOT="Data_Loader/Data/evalsrc" \
+    CKPT_EXTRA="$EX" CKPTS="t32 p1final base680" TASKS="$TK" N=100 \
+    T=$T YARN=$S YARN_ORIG=4096 PROMPT_LEN=$PL GPUS=0,1 NP=1 OUT="$O" \
+      ./run_longeval.sh >> "$L/ev_${TG}_${PL}.log" 2>&1
+    if [ -s "$O/summary.csv" ] && [ "$(wc -l < "$O/summary.csv")" -gt 1 ]; then
+      say "    $(head -1 $O/summary.csv)"; tail -n +2 "$O/summary.csv"|sed 's/^/      /'|tee -a "$M" >&2
+    else say "    FAILED"; fi
+  }
+  say "=== EVAL: three-way, t32 vs p1final vs base680 ==="
+  ev 131072 32 32744 niah_single x32
+  ev 262144 64 32744 niah_single x64
+  ev 262144 64 65512 niah_single x64
+  ev 262144 64 32744 "val_backward niah_multikey" vb
+fi
+
+up(){ [ -f "$1" ] || { say "  MISSING $1"; return 1; }
+  hf upload "$REPO" "$1" "$2" >>"$L/upload.log" 2>&1
+  python - "$REPO" "$1" "$2" <<'V'
+import sys,os
+from huggingface_hub import HfApi
+w=os.path.getsize(sys.argv[2])
+g={f.rfilename:(f.size or 0) for f in HfApi().repo_info(sys.argv[1],files_metadata=True).siblings}.get(sys.argv[3])
+ok=(g==w); print(("  VERIFIED  " if ok else "  *** FAILED ")+f"{sys.argv[3]} local={w:,} remote={g}")
+sys.exit(0 if ok else 1)
+V
+}
+BAD=0
+say "=== UPLOAD ==="
+for f in t32_train.sh t32_finish.sh final_evals.sh memprobe.sh dl_t32.sh; do
+  up "$f" "scripts/$f" || BAD=$((BAD+1)); done
+up Data_Loader/Data/t32mix/MANIFEST.txt data/t32mix_MANIFEST.txt || BAD=$((BAD+1))
+for g in "$L"/master.log "$L"/train.log "$L"/probe.log; do
+  [ -f "$g" ] && { cp -f "$g" "${g}.snap"; up "${g}.snap" "logs/t32_$(basename $g)" || BAD=$((BAD+1)); }; done
+for d in t32ev_* fin_*; do [ -f "$d/summary.csv" ] || continue
+  up "$d/summary.csv" "evals/2026-09-16/${d}_summary.csv" || BAD=$((BAD+1)); done
+if [ -n "$FINAL" ]; then
+  say "=== final checkpoint, both ranks (full resumable) ==="
+  ST=$(basename "$FINAL" | grep -oE 'step_[0-9]+' | cut -d_ -f2)
+  for r in 0 1; do for e in ckpt zero_g1; do
+    f="checkpoints_bluscriptcp/rank_$r/$(basename "${FINAL%.ckpt}").$e"
+    f="checkpoints_bluscriptcp/rank_$r/blumodelcp_run${RN}_step_${ST}.$e"
+    [ -f "$f" ] && { up "$f" "checkpoints/t32_680m_step${ST}_rank${r}.$e" || BAD=$((BAD+1)); }
+  done; done
+fi
+say "=== TEARDOWN VERDICT ==="
+if [ "$BAD" -eq 0 ] && [ -n "$FINAL" ]; then
+  say "ALL VERIFIED -- SAFE TO TEAR DOWN"
+else
+  say "*** $BAD failure(s), final='${FINAL:-none}' -- DO NOT TEAR DOWN ***"
+fi