Upload scripts/teardown_upload2.sh with huggingface_hub

scripts/teardown_upload2.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Teardown upload for the 2026-09-15 session. Every file is confirmed by the byte
+# size HF reports back -- an "ok" from the CLI is not proof (2026-09-10: it
+# returned SUCCESS for a missing file and 17 GB was lost silently).
+set -uo pipefail
+cd "$(dirname "$0")"
+source /root/Adhi/BluTrain/envv/bin/activate
+HF=hf; PY=python; REPO=unparallelled/BluScriptCP
+F=mixcont_logs/teardown2.log; : > "$F"
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$F" >&2; }
+MAN=/tmp/td2_manifest.txt; : > "$MAN"
+
+up(){ # src dst
+  local src=$1 dst=$2
+  [ -e "$src" ] || { say "  MISSING (not uploaded): $src"; echo "MISSING|$src|$dst" >>"$MAN"; return 1; }
+  $HF upload "$REPO" "$src" "$dst" >>mixcont_logs/upload2.log 2>&1 \
+    || { say "  UPLOAD FAILED $dst"; echo "FAILED|$src|$dst" >>"$MAN"; return 1; }
+  [ -f "$src" ] && echo "FILE|$src|$dst" >>"$MAN" || echo "DIR|$src|$dst" >>"$MAN"
+  say "  sent $dst"
+}
+
+say "=== 1. checkpoints (the branch points) ==="
+up checkpoints_bluscriptcp/rank_0/cliff_g1400.ckpt   checkpoints/mixcont_g1400_cliff.ckpt
+# mix64k_step1200 and mixcont_step200 are already on HF from earlier runs
+
+say "=== 2. the PPL-vs-position evidence (how the collapse was diagnosed) ==="
+up mixcont_logs/ppl_clean1200.csv    logs/mixcont/ppl_clean_g1200.csv
+up mixcont_logs/ppl_degraded1400.csv logs/mixcont/ppl_degraded_g1400.csv
+up mixcont_logs/ppl_clean1200.log    logs/mixcont/ppl_clean_g1200.log
+up mixcont_logs/ppl_degraded1400.log logs/mixcont/ppl_degraded_g1400.log
+
+say "=== 3. the g1400 eval (proof the loss cliff cost capability) ==="
+for d in cliff1400_len_*; do [ -d "$d" ] && up "$d" "logs/mixcont/$d/"; done
+up mixcont_logs/eval.log logs/mixcont/eval.log
+
+say "=== 4. training logs ==="
+up mixcont_logs/train.log   logs/mixcont/train.log
+up mixcont_logs/master.log  logs/mixcont/master.log
+up mixcont_logs/probe.log   logs/mixcont/probe.log
+up mixcont_logs/shipper.log logs/mixcont/shipper.log
+
+say "=== 5. scripts + the data manifest (so this is reproducible) ==="
+for s in mixcont_h100.sh ckpt_shipper.sh eval_cliff.sh ppl_vs_pos.sh \
+         mk_mix64k_plots.py mix_buckets.py dcpm_index.py teardown_upload2.sh; do
+  [ -f "$s" ] && up "$s" "scripts/$s"
+done
+up Data_Loader/Data/mixcont/MANIFEST.txt docs/mixcont_MANIFEST.txt
+
+say "=== 6. VERIFY every upload against the size HF reports back ==="
+$PY - "$REPO" "$MAN" <<'PYEOF' | tee -a "$F"
+import sys, os
+from huggingface_hub import HfApi
+repo, man = sys.argv[1], sys.argv[2]
+remote={f.rfilename:(f.size or 0) for f in HfApi().repo_info(repo,files_metadata=True).siblings}
+rows=[l.rstrip("\n").split("|",2) for l in open(man) if l.strip()]
+bad=0
+for kind,src,dst in rows:
+    if kind in ("MISSING","FAILED"): print(f"  {kind:<8} {dst}"); bad+=1; continue
+    if kind=="DIR":                  print(f"  DIR      {dst} (not size-checked)"); continue
+    want=os.path.getsize(src); got=remote.get(dst)
+    if got==want: print(f"  VERIFIED {want:>14,}  {dst}")
+    else:         print(f"  *** {'ABSENT' if got is None else 'MISMATCH'} local={want:,} remote={got}  {dst}"); bad+=1
+print(f"\n{len(rows)-bad}/{len(rows)} verified")
+sys.exit(1 if bad else 0)
+PYEOF
+VOK=$?
+[ "$VOK" -eq 0 ] && say "=== ALL VERIFIED -- safe to tear down ===" \
+                 || say "=== VERIFICATION FAILED -- DO NOT TEAR DOWN ==="
+exit $VOK