File size: 2,100 Bytes
7717fda
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
#!/usr/bin/env bash
# =============================================================================
# parallel.sh - saturate the GPU with several training stages at once.
#
# One process per stage, each with its own run id, checkpoint cadence and Hub
# push schedule; each resumes from its own last checkpoint. Sized for a ~98 GB
# card: captions (real photos) + voice + vocoder + ear together use roughly half
# of it, leaving headroom for the allocator and for a movie render.
#
#   nohup bash scripts/parallel.sh > /marimo/logs/parallel.log 2>&1 &
#   bash scripts/parallel.sh --stop
# =============================================================================
set -u
cd /marimo
PIDFILE=/tmp/parallel.pids
PRESET=${PRESET:-xl}

if [ "${1:-}" = "--stop" ]; then
  [ -f "$PIDFILE" ] && xargs -r kill -TERM < "$PIDFILE"
  echo "[parallel] stopped"; exit 0
fi

: > "$PIDFILE"
worker () {                       # stage steps batch lr
  local stage=$1 steps=$2 batch=$3 lr=$4
  while :; do
    echo "[parallel] $(date '+%T') start $stage batch=$batch steps=$steps"
    PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
    HF_HUB_DISABLE_PROGRESS_BARS=1 \
    python -m omni.cli train --stage "$stage" --preset "$PRESET" --steps "$steps" \
      --batch "$batch" --lr "$lr" --run_tag main \
      --set ckpt_steps=500 --set ckpt_seconds=1200 --set push_seconds=2400 \
      --set workers=4 --set log_every=50 >> "/marimo/logs/${stage}.log" 2>&1
    rc=$?
    echo "[parallel] $(date '+%T') $stage exited rc=$rc (resumes from last checkpoint)"
    sleep 45
  done
}

# the corpus decode must finish before a trainer touches the memmap; the index
# file only exists once prepare() completed, so gate on it
( while [ ! -f /marimo/data/real/captions.json ]; do sleep 15; done
  echo "[parallel] corpus ready"; worker captions 200000 192 3e-4 ) & echo $! >> "$PIDFILE"
worker speech    20000   96 2e-4 & echo $! >> "$PIDFILE"
worker vocoder   30000   32 1.5e-4 & echo $! >> "$PIDFILE"
worker ear       30000   48 2e-4 & echo $! >> "$PIDFILE"
echo "[parallel] workers: $(cat $PIDFILE | tr '\n' ' ')"
wait