Upload scripts/t128d_ship.sh with huggingface_hub

scripts/t128d_ship.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Ship every t128d checkpoint to HF BEFORE the trainer's KEEP=4 rotation can
+# delete it. Rotation drops a save 200 steps (~53 min) behind the newest, so
+# there is ample slack -- but nothing was shipping until this ran.
+#
+#   rank_0 .ckpt  for EVERY save   (2.1 GB, enough to EVALUATE)
+#   full 16-file  at MILESTONES    (17.6 GB, enough to RESUME)
+set -uo pipefail
+cd "$(dirname "$0")"
+PY=/root/Adhi/BluTrain/envv/bin/python
+REPO=unparallelled/BluScriptCP
+RUN=${RUN:-2}; PREFIX=blumodelcp; CKD=checkpoints_bluscriptcp
+L=t128d_logs; mkdir -p "$L"; S="$L/shipped.txt"; touch "$S"
+MILESTONES=" 300 360 "
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$L/ship.log" >&2; }
+
+up(){ # local_path  remote_name
+  grep -qxF "$2" "$S" && return 0
+  [ -s "$1" ] || return 1
+  $PY - "$1" "$2" "$REPO" <<'PY' || return 1
+import sys
+from huggingface_hub import HfApi
+HfApi().upload_file(path_or_fileobj=sys.argv[1],
+                    path_in_repo="checkpoints/"+sys.argv[2],
+                    repo_id=sys.argv[3])
+PY
+  echo "$2" >> "$S"; say "  shipped $2 ($(du -h "$1"|cut -f1))"
+}
+
+say "=== t128d shipper: rank_0 every save, full set at${MILESTONES}==="
+while :; do
+  # newest step present anywhere
+  STEPS=$(ls $CKD/rank_0/${PREFIX}_run${RUN}_step_*.ckpt 2>/dev/null \
+          | sed -E 's/.*_step_([0-9]+)\.ckpt/\1/' | sort -n)
+  for s in $STEPS; do
+    up "$CKD/rank_0/${PREFIX}_run${RUN}_step_${s}.ckpt" "t128d_step${s}_rank0.ckpt"
+    case "$MILESTONES" in *" $s "*)
+      for r in 0 1 2 3 4 5 6 7; do
+        up "$CKD/rank_${r}/${PREFIX}_run${RUN}_step_${s}.ckpt"   "t128d_step${s}_rank${r}.ckpt"
+        up "$CKD/rank_${r}/${PREFIX}_run${RUN}_step_${s}.zero_g1" "t128d_step${s}_rank${r}.zero_g1"
+      done
+      say "  MILESTONE $s: full resumable set up";;
+    esac
+  done
+  # stop once the trainer is gone AND step 600 is shipped
+  if ! ps -eo args | grep -qE '^[^ ]*build/bluscriptCP_exec'; then
+    grep -qxF "t128d_step600_rank0.ckpt" "$S" && { say "=== SHIPPER DONE ($(wc -l < "$S") files) ==="; exit 0; }
+    sleep 60
+    ps -eo args | grep -qE '^[^ ]*build/bluscriptCP_exec' || { say "=== trainer gone; shipped $(wc -l < "$S") files ==="; exit 0; }
+  fi
+  sleep 45
+done