Upload scripts/p1_finish.sh with huggingface_hub

scripts/p1_finish.sh

@@ -0,0 +1,140 @@
+#!/usr/bin/env bash
+# =============================================================================
+# End-of-session: wait for phase 1 -> NIAH eval -> upload everything -> verdict.
+# Leaves the box ready for teardown. Does NOT start phase 2.
+#
+# EVAL DESIGN
+#   task/N/haystack are IDENTICAL to the prior 1.06B evals (niah_single, N=100,
+#   b2 val_shard_00000) so the numbers sit on the same axis as 83/70/50/31.
+#   Two prompt lengths only -- 8192 and 16360. The phase-1 model has a 16k
+#   window, so 32744/65512 would measure a window it was never trained for.
+#   base680 (varybatch, un-extended) runs the same eval as the "before": it
+#   shows what the extension bought vs zero-shot YaRN.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+source /root/Adhi/BluTrain/envv/bin/activate 2>/dev/null
+REPO=unparallelled/BluScriptCP
+L=p1_logs; M="$L/finish.log"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$M" >&2; }
+
+# ---- 1. wait for training ---------------------------------------------------
+say "waiting for phase 1 to finish..."
+while ! grep -qa 'PHASE 1 TRAINING COMPLETE\|FATAL' "$L/master.log" 2>/dev/null; do
+  pgrep -f bluscriptCP_exec >/dev/null 2>&1 || { sleep 20
+    grep -qa 'PHASE 1 TRAINING COMPLETE\|FATAL' "$L/master.log" 2>/dev/null || {
+      say "trainer gone without COMPLETE -- aborting eval"; exit 1; }; }
+  sleep 30
+done
+grep -qa FATAL "$L/master.log" && { say "phase 1 FAILED -- no eval"; exit 1; }
+say "phase 1 complete: $(grep -a 'training done' "$L/master.log" | tail -1)"
+
+RN=$(cat "$L/runnum")
+FINAL=$(ls -t checkpoints_bluscriptcp/rank_0/blumodelcp_run${RN}_step_*.ckpt | head -1)
+FSTEP=$(basename "$FINAL" | grep -oE 'step_[0-9]+' | cut -d_ -f2)
+say "final checkpoint: $(basename "$FINAL")  ($(stat -c%s "$FINAL") bytes)"
+
+# ---- 2. eval ----------------------------------------------------------------
+A1B_680="CP_N_EMBD=1536 CP_N_LAYER=24 CP_N_HEAD=12 CP_N_KVHEAD=4 CP_FFN=4096 CP_WEIGHT_TYING=1"
+HAYSTACK="Data_Loader/Data/b2_8192_16384/val_shard_00000.bin"
+[ -f "$HAYSTACK" ] || { say "FATAL: haystack missing"; exit 1; }
+for PL in 8192 16360; do
+  O="p1_eval_len${PL}"
+  [ -f "$O/state/DONE" ] && { say "len $PL already done"; continue; }
+  say "eval prompt_len=$PL (p1final @step $FSTEP + base680 baseline), 2 GPUs"
+  A1B="$A1B_680" HAY="$HAYSTACK" \
+  CKPT_EXTRA="p1final=$(basename "$FINAL") base680=varybatch.ckpt" \
+  CKPTS="p1final base680" TASKS=niah_single N=100 \
+  T=16384 YARN=4 YARN_ORIG=4096 PROMPT_LEN=$PL \
+  GPUS=0,1 NP=1 OUT="$O" \
+    ./run_longeval.sh >> "$L/eval_${PL}.log" 2>&1
+  if [ -s "$O/summary.csv" ]; then mkdir -p "$O/state"; touch "$O/state/DONE"
+    say "  len $PL OK"; else say "  len $PL PRODUCED NO SUMMARY -- see $L/eval_${PL}.log"; fi
+done
+
+# ---- 3. results -------------------------------------------------------------
+say "=== PHASE 1 RESULTS (niah_single, N=100, same haystack as the 1.06B runs) ==="
+python - <<'PY' 2>&1 | tee -a "$M"
+import csv,os
+for pl in (8192,16360):
+    f=f"p1_eval_len{pl}/summary.csv"
+    if not os.path.exists(f): print(f"  len {pl}: NO RESULT"); continue
+    for r in csv.DictReader(open(f)):
+        k={x.lower():y for x,y in r.items()}
+        print(f"  len {pl:5d}  {r}")
+PY
+grep -ahA7 '=== niah_single' "$L"/eval_*.log 2>/dev/null | grep -aE 'accuracy|===' | tail -8 | tee -a "$M"
+
+# ---- 4. upload everything, VERIFY every byte --------------------------------
+up(){ [ -f "$1" ] || { say "  MISSING $1"; return 1; }
+  hf upload "$REPO" "$1" "$2" >>"$L/upload.log" 2>&1
+  python - "$REPO" "$1" "$2" <<'PY'
+import sys,os
+from huggingface_hub import HfApi
+w=os.path.getsize(sys.argv[2])
+g={f.rfilename:(f.size or 0) for f in HfApi().repo_info(sys.argv[1],files_metadata=True).siblings}.get(sys.argv[3])
+ok=(g==w); print(("  VERIFIED  " if ok else "  *** FAILED ")+f"{sys.argv[3]} local={w:,} remote={g}")
+sys.exit(0 if ok else 1)
+PY
+}
+verify_remote(){ # local remote -- confirm HF reports the same byte size
+  python - "$REPO" "$1" "$2" <<'VEOF'
+import sys,os
+from huggingface_hub import HfApi
+w=os.path.getsize(sys.argv[2])
+g={f.rfilename:(f.size or 0) for f in HfApi().repo_info(sys.argv[1],files_metadata=True).siblings}.get(sys.argv[3])
+ok=(g==w); print(("  VERIFIED  " if ok else "  *** FAILED ")+f"{sys.argv[3]} local={w:,} remote={g}")
+sys.exit(0 if ok else 1)
+VEOF
+}
+say "=== uploading ==="
+BAD=0
+for f in p1_680m.sh p2_680m.sh p1_shipper.sh p1_finish.sh dl_p23.sh run_longeval.sh \
+         mix_buckets.py HANDOFF_H200_32K.md; do up "$f" "scripts/$f" || BAD=$((BAD+1)); done
+up Scripts/Blutrain/bluscriptCP.cpp src/bluscriptCP.cpp || BAD=$((BAD+1))
+for m in p1mix p2mix p3mix; do up "Data_Loader/Data/$m/MANIFEST.txt" "data/${m}_MANIFEST.txt" || BAD=$((BAD+1)); done
+for g in "$L"/*.log "$L"/master.log; do [ -f "$g" ] && { up "$g" "logs/p1_$(basename "$g")" || BAD=$((BAD+1)); }; done
+for pl in 8192 16360; do
+  for f in p1_eval_len${pl}/summary.csv p1_eval_len${pl}/depth_summary.csv; do
+    [ -f "$f" ] && { up "$f" "evals/p1_len${pl}_$(basename "$f")" || BAD=$((BAD+1)); }; done
+  for f in p1_eval_len${pl}/scores/*.csv; do
+    [ -f "$f" ] && { up "$f" "evals/p1_len${pl}_$(basename "$f")" || BAD=$((BAD+1)); }; done
+done
+up "vault_out/ClaudeReport_session_2026-09-15_680m_tied_head_fix_and_phase1_extension.md" \
+   "docs/ClaudeReport_session_2026-09-15_680m_tied_head_fix_and_phase1_extension.md" || BAD=$((BAD+1))
+
+# ---- 5. final checkpoint: FULL resumable, both ranks -------------------------
+# The shipper already uploads the final step as a full resumable save (it is in
+# EVAL_STEPS). Re-uploading under a second name would push ~11 GB twice, so wait
+# for it, then only fill in what it did not get.
+say "=== final checkpoint (full resumable, both ranks) ==="
+for i in $(seq 1 60); do [ -f "$L/shipped_${FSTEP}.ok" ] && break; sleep 30; done
+if [ -f "$L/shipped_${FSTEP}.ok" ] && [ "$(cat "$L/shipped_${FSTEP}.kind" 2>/dev/null)" = full ]; then
+  say "  shipper shipped step $FSTEP as FULL resumable -- verifying, not re-uploading"
+  for r in 0 1; do for ext in ckpt zero_g1; do
+    f="checkpoints_bluscriptcp/rank_$r/blumodelcp_run${RN}_step_${FSTEP}.$ext"
+    [ -f "$f" ] || continue
+    verify_remote "$f" "checkpoints/p1_680m_step${FSTEP}_rank${r}.$ext" || BAD=$((BAD+1))
+  done; done
+else
+  say "  shipper did NOT complete step $FSTEP -- uploading it here"
+  for r in 0 1; do for ext in ckpt zero_g1; do
+    f="checkpoints_bluscriptcp/rank_$r/blumodelcp_run${RN}_step_${FSTEP}.$ext"
+    [ -f "$f" ] || continue
+    up "$f" "checkpoints/p1_680m_final_step${FSTEP}_rank${r}.$ext" || BAD=$((BAD+1))
+  done; done
+fi
+
+# stop the shipper so nothing is mid-flight at teardown
+SP=$(cat "$L/shipper.pid" 2>/dev/null)
+[ -n "$SP" ] && kill "$SP" 2>/dev/null && say "shipper stopped (pid $SP)"
+
+# ---- 6. verdict -------------------------------------------------------------
+say "=== TEARDOWN VERDICT ==="
+if [ "$BAD" -eq 0 ]; then
+  say "ALL UPLOADS VERIFIED ($REPO) -- SAFE TO TEAR DOWN"
+else
+  say "*** $BAD UPLOAD(S) FAILED -- DO NOT TEAR DOWN ***"
+fi
+say "done"