#!/usr/bin/env bash # ============================================================================= # parallel.sh - saturate the GPU with several training stages at once. # # One process per stage, each with its own run id, checkpoint cadence and Hub # push schedule; each resumes from its own last checkpoint. Sized for a ~98 GB # card: captions (real photos) + voice + vocoder + ear together use roughly half # of it, leaving headroom for the allocator and for a movie render. # # nohup bash scripts/parallel.sh > /marimo/logs/parallel.log 2>&1 & # bash scripts/parallel.sh --stop # ============================================================================= set -u cd /marimo PIDFILE=/tmp/parallel.pids PRESET=${PRESET:-xl} if [ "${1:-}" = "--stop" ]; then [ -f "$PIDFILE" ] && xargs -r kill -TERM < "$PIDFILE" echo "[parallel] stopped"; exit 0 fi : > "$PIDFILE" worker () { # stage steps batch lr local stage=$1 steps=$2 batch=$3 lr=$4 while :; do echo "[parallel] $(date '+%T') start $stage batch=$batch steps=$steps" PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ HF_HUB_DISABLE_PROGRESS_BARS=1 \ python -m omni.cli train --stage "$stage" --preset "$PRESET" --steps "$steps" \ --batch "$batch" --lr "$lr" --run_tag main \ --set ckpt_steps=500 --set ckpt_seconds=1200 --set push_seconds=2400 \ --set workers=4 --set log_every=50 >> "/marimo/logs/${stage}.log" 2>&1 rc=$? echo "[parallel] $(date '+%T') $stage exited rc=$rc (resumes from last checkpoint)" sleep 45 done } # the corpus decode must finish before a trainer touches the memmap; the index # file only exists once prepare() completed, so gate on it ( while [ ! -f /marimo/data/real/captions.json ]; do sleep 15; done echo "[parallel] corpus ready"; worker captions 200000 192 3e-4 ) & echo $! >> "$PIDFILE" worker speech 20000 96 2e-4 & echo $! >> "$PIDFILE" worker vocoder 30000 32 1.5e-4 & echo $! >> "$PIDFILE" worker ear 30000 48 2e-4 & echo $! >> "$PIDFILE" echo "[parallel] workers: $(cat $PIDFILE | tr '\n' ' ')" wait