Upload source/files/upload_all_to_hf.sh with huggingface_hub

source/files/upload_all_to_hf.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Upload every artifact of the IN2 / reach work to HuggingFace.
+# Idempotent: re-running re-uploads changed files and skips identical ones.
+# Does NOT handle git or the vault logs branch -- see the note at the end.
+set -uo pipefail
+cd "$(dirname "$0")"
+
+HF=/root/Adhi/BluTrain/envv/bin/hf
+REPO=unparallelled/BluScriptCP
+CK=checkpoints_bluscriptcp/rank_0
+ok=0; fail=0
+up(){ # up <local> <path-in-repo> <label>
+  if [ ! -e "$1" ]; then echo "  SKIP (absent): $3"; return; fi
+  if "$HF" upload "$REPO" "$1" "$2" >/dev/null 2>&1; then
+    echo "  ok   $3"; ok=$((ok+1))
+  else
+    echo "  FAIL $3"; fail=$((fail+1))
+  fi
+}
+
+echo "=== 1. source ==="
+tar czf /tmp/cp_source_now.tgz \
+  --exclude='build' --exclude='checkpoints_bluscriptcp' --exclude='in2_data' \
+  --exclude='Data_Loader/Data' --exclude='.git' --exclude='*.bin' --exclude='probe_*' \
+  -C /root/Adhi/BluTrain dist/Context_Parallelism dist/zero 2>/dev/null
+up /tmp/cp_source_now.tgz "source/cp_source_$(date -u +%Y-%m-%d).tgz" "source tarball"
+for f in run_in2_experiment.sh run_probepos_experiment.sh upload_all_to_hf.sh probe_1b.sh \
+         Tests/bluscriptcp/needle_score.py Tests/bluscriptcp/needle_gen.py \
+         Tests/bluscriptcp/in2_gen.py Tests/bluscriptcp/in2_gate.py \
+         Tests/bluscriptcp/needle_sweep.sh; do
+  up "$f" "source/files/$(basename "$f")" "$(basename "$f")"
+done
+
+echo "=== 2. checkpoints (3.9 GB each -- the slow part) ==="
+up "$CK/blumodelcp_run7_step_750.ckpt"  checkpoints/in2/extended_blumodelcp_run7_step_750.ckpt "base run7 (init source)"
+up "$CK/blumodelcp_run12_step_400.ckpt" checkpoints/in2/arm_a_blumodelcp_run12_step_400.ckpt  "arm A run12"
+up "$CK/blumodelcp_run15_step_400.ckpt" checkpoints/in2/arm_b_blumodelcp_run15_step_400.ckpt  "arm B run15"
+up "$CK/blumodelcp_run16_step_400.ckpt" checkpoints/in2/arm_c_blumodelcp_run16_step_400.ckpt  "arm C run16"
+
+echo "=== 3. probe results (reach.csv + per-job CSVs + needles + state) ==="
+for d in probe_arm_a probe_arm_b probe_arm_c probe_run7_yarn4 probe_run8_s4 \
+         probe_pos8192_base probe_pos8192_armB probe_pos8192_armC \
+         probe_pos12288_base probe_pos12288_armB probe_pos12288_armC; do
+  up "$d" "logs/$d/" "$d"
+done
+
+echo "=== 4. training CSVs and configs ==="
+up CP_BluScreipt_Training_logs logs/in2_train_csv/ "training CSVs"
+
+echo "=== 5. run logs ==="
+up in2_logs        logs/in2/           "driver + arm training logs"
+up probepos_logs   logs/probepos_logs/ "probe-pos master log"
+up run_logs        logs/in2_stdout/    "stdout logs"
+
+echo "=== 6. vault snapshot (reports + log index) ==="
+up /root/claude-vault docs/ "vault"
+
+echo
+echo "uploaded: $ok   failed: $fail"
+echo
+echo "NOT covered by this script:"
+echo "  - git commits/pushes for BluTrain and the Context_Parallelism submodule"
+echo "  - the vault logs branch: run scripts/logs-push.sh so today's log lines and"
+echo "    report reach the laptop; the docs/ copy above is a snapshot, not the branch"
+echo "  - in2_data/ (1.4 GB of generated shards) is deliberately excluded:"
+echo "    regenerate deterministically with in2_gen.py --seed 0"