File size: 1,985 Bytes
32f554b 88b3424 32f554b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | #!/usr/bin/env bash
# Start/stop/inspect the Hugging Face checkpoint auto-sync (scripts/hf_ckpt_sync.py) on the GPU pod.
# bash scripts/hf_ckpt_sync.sh start # start the 3-hourly loop if not already running (safe to re-run, e.g. after a pod restart)
# bash scripts/hf_ckpt_sync.sh status # pid, last log lines, last uploaded file
# bash scripts/hf_ckpt_sync.sh once # one immediate sync cycle (fails if the loop is mid-upload, lock-protected)
# bash scripts/hf_ckpt_sync.sh stop
# No cron on the pod: after a pod restart run "start" again (the training driver must be restarted separately).
set -u
cd "$(dirname "$0")/.."
PY=${HFSYNC_PY:-$HOME/hfsync-venv/bin/python}; PIDF=logs/hf_sync.pid; LOG=logs/hf_sync.log
mkdir -p logs
running() { [ -f "$PIDF" ] && kill -0 "$(cat "$PIDF")" 2>/dev/null; }
case "${1:-status}" in
start) if running; then echo "already running pid $(cat $PIDF)"; exit 0; fi
[ -s "$HOME/.cache/huggingface/token" ] || { echo "no HF token at ~/.cache/huggingface/token"; exit 1; }
[ -x "$PY" ] || { /opt/conda/bin/python -m venv "$HOME/hfsync-venv" && "$HOME/hfsync-venv/bin/pip" install -q -U huggingface_hub; }
SETSID=""; command -v setsid >/dev/null && SETSID=setsid
IONICE=""; command -v ionice >/dev/null && IONICE="ionice -c3"
$SETSID nohup nice -n 19 $IONICE "$PY" -u scripts/hf_ckpt_sync.py --loop >> "$LOG" 2>&1 < /dev/null &
sleep 3; running && echo "started pid $(cat $PIDF), log $LOG" || { echo "failed to start; see $LOG"; tail -5 "$LOG"; exit 1; } ;;
stop) running && kill "$(cat $PIDF)" && echo stopped || echo "not running" ;;
once) nice -n 19 ionice -c3 "$PY" -u scripts/hf_ckpt_sync.py >> "$LOG" 2>&1; tail -3 "$LOG" ;;
status) running && echo "running pid $(cat $PIDF)" || echo "NOT running (start: bash scripts/hf_ckpt_sync.sh start)"
tail -n 5 "$LOG" 2>/dev/null; cat logs/hf_sync_state.json 2>/dev/null; echo ;;
*) sed -n '2,7p' "$0"; exit 1 ;;
esac
|