| # Start all eight vimeml-tiny-zh-v1 arms on one node, one GPU each, detached from this shell. | |
| # GPUs default to 0-7; override with VIMEML_ZH13_GPUS="0 1 2 3 4 5 6 7". | |
| # Rerunning resumes arms whose process has stopped and skips arms still running. | |
| set -euo pipefail | |
| cd "$(dirname "$0")/../.." | |
| arms=(12k-fresh-s1 12k-fresh-s2 12k-fresh-s3 12k-ep5 12k-ep20 8k-fresh 8k-ep5 8k-ep20) | |
| read -r -a gpus <<< "${VIMEML_ZH13_GPUS:-0 1 2 3 4 5 6 7}" | |
| (( ${#gpus[@]} == ${#arms[@]} )) || { echo "Need ${#arms[@]} GPUs, got ${#gpus[@]}" >&2; exit 2; } | |
| mkdir -p logs/zh13 | |
| for i in "${!arms[@]}"; do | |
| arm="${arms[$i]}" | |
| pidfile="logs/zh13/${arm}.pid" | |
| if [[ -f "$pidfile" ]] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then | |
| echo "$arm already running (pid $(cat "$pidfile"))"; continue | |
| fi | |
| setsid nohup bash scripts/training/cluster_zh13.sh "$arm" "${gpus[$i]}" \ | |
| >> "logs/zh13/${arm}.log" 2>&1 < /dev/null & | |
| echo $! > "$pidfile" | |
| echo "$arm -> GPU ${gpus[$i]}, pid $!, log logs/zh13/${arm}.log" | |
| done | |