Upload scripts/ship_live.sh with huggingface_hub

scripts/ship_live.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Ship checkpoints AS THEY APPEAR, in parallel with training. Priority given the
+# 2h deadline: rank_0 of every save (enough for eval + CP_INIT_FROM branching,
+# since ZeRO replicates weights), then ALL 8 ranks of step 500 (a resume needs
+# every rank's .ckpt + .zero_g1). Verifies byte size; deletes nothing.
+set -uo pipefail; cd "$(dirname "$0")"
+export PATH=/root/Adhi/BluTrain/envv/bin:$PATH
+R=unparallelled/BluScriptCP; L=t128b_logs/ship.log; mkdir -p t128b_logs
+say(){ echo "[$(date -u +%H:%M:%S)] $*" | tee -a "$L" >&2; }
+up(){ # local remote
+  [ -f "$1" ] || return 1
+  hf upload "$R" "$1" "$2" >>"$L" 2>&1
+  python - "$R" "$1" "$2" <<'V'
+import sys,os
+from huggingface_hub import HfApi
+w=os.path.getsize(sys.argv[2])
+g={x.rfilename:(x.size or 0) for x in HfApi().repo_info(sys.argv[1],files_metadata=True).siblings}.get(sys.argv[3])
+sys.exit(0 if g==w else 1)
+V
+}
+say "live shipper armed"
+while true; do
+  RN=$(cat t128b_logs/runnum 2>/dev/null || ls checkpoints_bluscriptcp/rank_0/ 2>/dev/null | grep -oE 'run[0-9]+' | sort -u | tail -1 | tr -d 'run')
+  for s in 100 200 300 400 500; do
+    f="checkpoints_bluscriptcp/rank_0/blumodelcp_run${RN}_step_${s}.ckpt"
+    [ -f "$f" ] || continue
+    [ -f "t128b_logs/.r0_${s}.ok" ] && continue
+    sleep 8   # let the write flush
+    if up "$f" "checkpoints/t128b_step${s}_rank0.ckpt"; then
+      touch "t128b_logs/.r0_${s}.ok"; say "VERIFIED step $s rank0"
+    else say "retry step $s rank0"; fi
+  done
+  # full resumable set for step 500 once it exists on all ranks
+  if [ ! -f t128b_logs/.full500.ok ] && \
+     [ "$(ls checkpoints_bluscriptcp/rank_*/blumodelcp_run${RN}_step_500.ckpt 2>/dev/null|wc -l)" -ge 8 ]; then
+    sleep 10; ok=1
+    for r in 0 1 2 3 4 5 6 7; do for e in ckpt zero_g1; do
+      up "checkpoints_bluscriptcp/rank_${r}/blumodelcp_run${RN}_step_500.${e}" \
+         "checkpoints/t128b_step500_rank${r}.${e}" || ok=0
+    done; done
+    [ "$ok" = 1 ] && { touch t128b_logs/.full500.ok; say "VERIFIED full resumable step 500 (8 ranks)"; }
+  fi
+  [ -f t128b_logs/.full500.ok ] && [ -f t128b_logs/.r0_500.ok ] && { say "SHIP COMPLETE"; break; }
+  grep -qa 'STOPPED AT 500' t128b_logs/master.log 2>/dev/null && [ -f t128b_logs/.full500.ok ] && break
+  sleep 45
+done