File size: 941 Bytes
81cffe7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | #!/usr/bin/env bash
# Start all six V4.4 arms on one node, one GPU each, detached from this shell.
# GPUs default to 0-5; override with VIMEML_V44_GPUS="2 3 4 5 6 7".
# Rerunning resumes arms whose process has stopped and skips arms still running.
set -euo pipefail
cd "$(dirname "$0")/../.."
arms=(s1 s2 s3 chat0 chat10 crop60)
read -r -a gpus <<< "${VIMEML_V44_GPUS:-0 1 2 3 4 5}"
(( ${#gpus[@]} == ${#arms[@]} )) || { echo "Need ${#arms[@]} GPUs, got ${#gpus[@]}" >&2; exit 2; }
mkdir -p logs/v44
for i in "${!arms[@]}"; do
arm="${arms[$i]}"
pidfile="logs/v44/${arm}.pid"
if [[ -f "$pidfile" ]] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then
echo "$arm already running (pid $(cat "$pidfile"))"; continue
fi
setsid nohup bash scripts/training/cluster_v44.sh "$arm" "${gpus[$i]}" \
>> "logs/v44/${arm}.log" 2>&1 < /dev/null &
echo $! > "$pidfile"
echo "$arm -> GPU ${gpus[$i]}, pid $!, log logs/v44/${arm}.log"
done
|