Download scripts/parallel.sh from hkblockchaintechnology/VLM-all-inclusive: direct link, hf CLI and curl.
- Browser
- Download file 2.1 kB
-
https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/parallel.sh
- Command line
-
hf download hf://hkblockchaintechnology/VLM-all-inclusive/scripts/parallel.sh
-
curl -L -o parallel.sh https://huggingface.co/hkblockchaintechnology/VLM-all-inclusive/resolve/main/scripts/parallel.sh
2.1 kB
| # ============================================================================= | |
| # parallel.sh - saturate the GPU with several training stages at once. | |
| # | |
| # One process per stage, each with its own run id, checkpoint cadence and Hub | |
| # push schedule; each resumes from its own last checkpoint. Sized for a ~98 GB | |
| # card: captions (real photos) + voice + vocoder + ear together use roughly half | |
| # of it, leaving headroom for the allocator and for a movie render. | |
| # | |
| # nohup bash scripts/parallel.sh > /marimo/logs/parallel.log 2>&1 & | |
| # bash scripts/parallel.sh --stop | |
| # ============================================================================= | |
| set -u | |
| cd /marimo | |
| PIDFILE=/tmp/parallel.pids | |
| PRESET=${PRESET:-xl} | |
| if [ "${1:-}" = "--stop" ]; then | |
| [ -f "$PIDFILE" ] && xargs -r kill -TERM < "$PIDFILE" | |
| echo "[parallel] stopped"; exit 0 | |
| fi | |
| : > "$PIDFILE" | |
| worker () { # stage steps batch lr | |
| local stage=$1 steps=$2 batch=$3 lr=$4 | |
| while :; do | |
| echo "[parallel] $(date '+%T') start $stage batch=$batch steps=$steps" | |
| PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ | |
| HF_HUB_DISABLE_PROGRESS_BARS=1 \ | |
| python -m omni.cli train --stage "$stage" --preset "$PRESET" --steps "$steps" \ | |
| --batch "$batch" --lr "$lr" --run_tag main \ | |
| --set ckpt_steps=500 --set ckpt_seconds=1200 --set push_seconds=2400 \ | |
| --set workers=4 --set log_every=50 >> "/marimo/logs/${stage}.log" 2>&1 | |
| rc=$? | |
| echo "[parallel] $(date '+%T') $stage exited rc=$rc (resumes from last checkpoint)" | |
| sleep 45 | |
| done | |
| } | |
| # the corpus decode must finish before a trainer touches the memmap; the index | |
| # file only exists once prepare() completed, so gate on it | |
| ( while [ ! -f /marimo/data/real/captions.json ]; do sleep 15; done | |
| echo "[parallel] corpus ready"; worker captions 200000 192 3e-4 ) & echo $! >> "$PIDFILE" | |
| worker speech 20000 96 2e-4 & echo $! >> "$PIDFILE" | |
| worker vocoder 30000 32 1.5e-4 & echo $! >> "$PIDFILE" | |
| worker ear 30000 48 2e-4 & echo $! >> "$PIDFILE" | |
| echo "[parallel] workers: $(cat $PIDFILE | tr '\n' ' ')" | |
| wait | |