VLM-all-inclusive / scripts /parallel.sh
hkblockchaintechnology's picture
docs: maximal design + WHY; omnivocoder GAN, real-data stage, parallel launcher
7717fda verified
Raw History Blame Contribute Delete
2.1 kB
#!/usr/bin/env bash
# =============================================================================
# parallel.sh - saturate the GPU with several training stages at once.
#
# One process per stage, each with its own run id, checkpoint cadence and Hub
# push schedule; each resumes from its own last checkpoint. Sized for a ~98 GB
# card: captions (real photos) + voice + vocoder + ear together use roughly half
# of it, leaving headroom for the allocator and for a movie render.
#
# nohup bash scripts/parallel.sh > /marimo/logs/parallel.log 2>&1 &
# bash scripts/parallel.sh --stop
# =============================================================================
set -u
cd /marimo
PIDFILE=/tmp/parallel.pids
PRESET=${PRESET:-xl}
if [ "${1:-}" = "--stop" ]; then
[ -f "$PIDFILE" ] && xargs -r kill -TERM < "$PIDFILE"
echo "[parallel] stopped"; exit 0
fi
: > "$PIDFILE"
worker () { # stage steps batch lr
local stage=$1 steps=$2 batch=$3 lr=$4
while :; do
echo "[parallel] $(date '+%T') start $stage batch=$batch steps=$steps"
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
HF_HUB_DISABLE_PROGRESS_BARS=1 \
python -m omni.cli train --stage "$stage" --preset "$PRESET" --steps "$steps" \
--batch "$batch" --lr "$lr" --run_tag main \
--set ckpt_steps=500 --set ckpt_seconds=1200 --set push_seconds=2400 \
--set workers=4 --set log_every=50 >> "/marimo/logs/${stage}.log" 2>&1
rc=$?
echo "[parallel] $(date '+%T') $stage exited rc=$rc (resumes from last checkpoint)"
sleep 45
done
}
# the corpus decode must finish before a trainer touches the memmap; the index
# file only exists once prepare() completed, so gate on it
( while [ ! -f /marimo/data/real/captions.json ]; do sleep 15; done
echo "[parallel] corpus ready"; worker captions 200000 192 3e-4 ) & echo $! >> "$PIDFILE"
worker speech 20000 96 2e-4 & echo $! >> "$PIDFILE"
worker vocoder 30000 32 1.5e-4 & echo $! >> "$PIDFILE"
worker ear 30000 48 2e-4 & echo $! >> "$PIDFILE"
echo "[parallel] workers: $(cat $PIDFILE | tr '\n' ' ')"
wait