File size: 1,012 Bytes
0cef53c 2f04f23 0cef53c 9b40da9 0cef53c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 | #!/bin/bash
# Data-parallel training on all GPUs of one or more nodes:
# sbatch -A <account> -N 2 scripts/train_ddp.sh [key=value ...]
# data.batch_size is per GPU. CUDA graphs need a single process.
#SBATCH -N 1
#SBATCH -C gpu
#SBATCH --ntasks-per-node=1
#SBATCH --gpus-per-node=4
#SBATCH -c 64
#SBATCH -t 02:00:00
#SBATCH -o logs/train_ddp_%j.out
set -euo pipefail
cd "${SLURM_SUBMIT_DIR:-$(pwd)}"
CONFIG="${CONFIG:-configs/gamgam.yaml}"
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
export UV_CACHE_DIR="${UV_CACHE_DIR:-${SCRATCH:-${HOME}/.cache}/uv}"
# Hugging Face downloads (the Delphes samples are 52 GB) go to scratch too.
export HF_HOME="${HF_HOME:-${SCRATCH:-${HOME}/.cache}/huggingface}"
MASTER_ADDR="$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | head -n 1)"
mkdir -p logs
srun uv run torchrun \
--nnodes="${SLURM_NNODES}" --nproc_per_node="${GPUS_PER_NODE:-4}" \
--rdzv_backend=c10d --rdzv_endpoint="${MASTER_ADDR}:${MASTER_PORT:-29500}" \
-m gnn4colliders.cli train "${CONFIG}" "$@"
|