File size: 2,100 Bytes
7717fda | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | #!/usr/bin/env bash
# =============================================================================
# parallel.sh - saturate the GPU with several training stages at once.
#
# One process per stage, each with its own run id, checkpoint cadence and Hub
# push schedule; each resumes from its own last checkpoint. Sized for a ~98 GB
# card: captions (real photos) + voice + vocoder + ear together use roughly half
# of it, leaving headroom for the allocator and for a movie render.
#
# nohup bash scripts/parallel.sh > /marimo/logs/parallel.log 2>&1 &
# bash scripts/parallel.sh --stop
# =============================================================================
set -u
cd /marimo
PIDFILE=/tmp/parallel.pids
PRESET=${PRESET:-xl}
if [ "${1:-}" = "--stop" ]; then
[ -f "$PIDFILE" ] && xargs -r kill -TERM < "$PIDFILE"
echo "[parallel] stopped"; exit 0
fi
: > "$PIDFILE"
worker () { # stage steps batch lr
local stage=$1 steps=$2 batch=$3 lr=$4
while :; do
echo "[parallel] $(date '+%T') start $stage batch=$batch steps=$steps"
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
HF_HUB_DISABLE_PROGRESS_BARS=1 \
python -m omni.cli train --stage "$stage" --preset "$PRESET" --steps "$steps" \
--batch "$batch" --lr "$lr" --run_tag main \
--set ckpt_steps=500 --set ckpt_seconds=1200 --set push_seconds=2400 \
--set workers=4 --set log_every=50 >> "/marimo/logs/${stage}.log" 2>&1
rc=$?
echo "[parallel] $(date '+%T') $stage exited rc=$rc (resumes from last checkpoint)"
sleep 45
done
}
# the corpus decode must finish before a trainer touches the memmap; the index
# file only exists once prepare() completed, so gate on it
( while [ ! -f /marimo/data/real/captions.json ]; do sleep 15; done
echo "[parallel] corpus ready"; worker captions 200000 192 3e-4 ) & echo $! >> "$PIDFILE"
worker speech 20000 96 2e-4 & echo $! >> "$PIDFILE"
worker vocoder 30000 32 1.5e-4 & echo $! >> "$PIDFILE"
worker ear 30000 48 2e-4 & echo $! >> "$PIDFILE"
echo "[parallel] workers: $(cat $PIDFILE | tr '\n' ' ')"
wait
|