Upload scripts/teardown_upload.sh with huggingface_hub

scripts/teardown_upload.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Teardown upload for the cont64k full-epoch run -- VERIFIED.
+#
+# Runs while PHASE 3 eval is still going, so the irreplaceable artifacts are
+# durable before anything can go wrong. It deliberately does NOT touch the
+# cont64k_len_* eval directories: those are being written right now, and the
+# script's own PHASE 4 uploads them when they are complete.
+#
+# WHY EVERY FILE IS SIZE-CHECKED AGAINST HF: on 2026-09-10 the upload helper
+# was `up(){ [ -e "$1" ] || return 0; ... }` -- it returned SUCCESS when the
+# source was missing. ~17 GB of checkpoints silently never left the box, the
+# CLI never errored, and the instance was torn down. Only a 61-byte pointer
+# file survived. An "ok" from the CLI is NOT proof; only the byte size that HF
+# reports back is.
+#
+# WHY ALL 8 RANKS: rank_0's .ckpt alone is enough to branch from (weights are
+# replicated, and CP_INIT_FROM is how every topology change in this project has
+# been done). But an EXACT resume at 8 ranks needs each rank's own .ckpt (it
+# carries that rank's ZeRO group-0 shard) AND its .zero_g1. After teardown
+# nothing is recoverable, so all 16 files go up -- 30.9 GB.
+# =============================================================================
+set -uo pipefail
+cd "$(dirname "$0")"
+source /root/gv/envv/bin/activate
+HF=/root/gv/envv/bin/hf
+PY=/root/gv/envv/bin/python3
+REPO=unparallelled/BluScriptCP
+CKD=checkpoints_bluscriptcp
+RUN=4; STEP=1651
+L=teardown_upload.log; : > "$L"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$L" >&2; }
+
+MANIFEST=/tmp/teardown_manifest.txt; : > "$MANIFEST"
+
+up(){ # src dst  -- records src/dst for the verification pass
+  local src=$1 dst=$2
+  if [ ! -e "$src" ]; then say "  MISSING SOURCE (NOT UPLOADED): $src"; echo "MISSING|$src|$dst" >> "$MANIFEST"; return 1; fi
+  local sz="-"; [ -f "$src" ] && sz=$(stat -c%s "$src")
+  say "  -> $dst ${sz}"
+  if $HF upload "$REPO" "$src" "$dst" >>"$L" 2>&1; then
+    [ -f "$src" ] && echo "FILE|$src|$dst" >> "$MANIFEST" || echo "DIR|$src|$dst" >> "$MANIFEST"
+  else
+    say "     CLI FAILED"; echo "FAILED|$src|$dst" >> "$MANIFEST"; return 1
+  fi
+}
+
+say "=== 1. the model (8 ranks x .ckpt + .zero_g1) ==="
+for r in 0 1 2 3 4 5 6 7; do
+  up "$CKD/rank_$r/blumodelcp_run${RUN}_step_${STEP}.ckpt"    "checkpoints/cont64k_b5_rank${r}_step_${STEP}.ckpt"
+  up "$CKD/rank_$r/blumodelcp_run${RUN}_step_${STEP}.zero_g1" "checkpoints/cont64k_b5_rank${r}_step_${STEP}.zero_g1"
+done
+
+say "=== 2. training logs + the script that produced them ==="
+up cont64k_logs/master.log "logs/cont64k/master.log"
+up cont64k_logs/cont.log   "logs/cont64k/cont.log"
+up cont64k_logs/probe.log  "logs/cont64k/probe.log"
+up cont64k_b5_h100.sh      "scripts/cont64k_b5_h100.sh"
+up "$(basename "$0")"      "scripts/$(basename "$0")"
+
+say "=== 3. VERIFY every upload against the size HF reports back ==="
+$PY - "$REPO" "$MANIFEST" <<'PYEOF' | tee -a "$L"
+import sys, os
+from huggingface_hub import HfApi
+repo, man = sys.argv[1], sys.argv[2]
+remote = {f.rfilename: (f.size or 0) for f in HfApi().repo_info(repo, files_metadata=True).siblings}
+rows = [l.rstrip("\n").split("|", 2) for l in open(man) if l.strip()]
+bad = 0
+for kind, src, dst in rows:
+    if kind in ("MISSING", "FAILED"):
+        print(f"  {kind:<9} {dst}"); bad += 1; continue
+    if kind == "DIR":
+        print(f"  DIR       {dst} (not size-checked)"); continue
+    want = os.path.getsize(src); got = remote.get(dst)
+    if got == want:
+        print(f"  VERIFIED  {want:>13,}  {dst}")
+    else:
+        print(f"  *** {'ABSENT' if got is None else 'MISMATCH'}  local={want:,} remote={got}  {dst}"); bad += 1
+print(f"\n{len(rows)-bad}/{len(rows)} verified")
+sys.exit(1 if bad else 0)
+PYEOF
+VOK=$?
+
+if [ "$VOK" -eq 0 ]; then
+  say "=== MODEL + LOGS VERIFIED ON HF ==="
+  say "Still pending before teardown: PHASE 4 uploads the eval results and"
+  say "RESULTS.md once eval finishes. Wait for its 'safe to tear down' line."
+else
+  say "=== VERIFICATION FAILED -- DO NOT TEAR DOWN ==="
+fi
+exit $VOK